add refresh logic

src/app/api/refresh-history/route.ts

@@ -0,0 +1,28 @@
+import { env } from '@/env'
+import { syncHistory } from '@/server/api/routers/history'
+import { headers } from 'next/headers'
+
+const SECURE_TOKEN = env.CRON_SECRET
+
+export async function POST() {
+  const headersList = await headers()
+  const authToken = headersList.get('authorization')?.replace('Bearer ', '')
+
+  if (authToken !== SECURE_TOKEN) {
+    return new Response('unauthorized', { status: 401 })
+  }
+
+  try {
+    try {
+      console.log('refreshing history...')
+      await syncHistory()
+    } catch (err) {
+      console.error('history refresh failed:', err)
+      return new Response('internal error', { status: 500 })
+    }
+    return Response.json({ success: true })
+  } catch (err) {
+    console.error('refresh failed:', err)
+    return new Response('internal error', { status: 500 })
+  }
+}

src/app/api/refresh-leaderboard/route.ts

@@ -0,0 +1,34 @@
+import { env } from '@/env'
+import { LeaderboardService } from '@/server/services/leaderboard'
+import { RANKED_CHANNEL, VANILLA_CHANNEL } from '@/shared/constants'
+import { headers } from 'next/headers'
+
+const SECURE_TOKEN = env.CRON_SECRET
+const CHANNEL_IDS = [RANKED_CHANNEL, VANILLA_CHANNEL]
+export async function POST() {
+  const headersList = await headers()
+  const authToken = headersList.get('authorization')?.replace('Bearer ', '')
+
+  if (authToken !== SECURE_TOKEN) {
+    return new Response('unauthorized', { status: 401 })
+  }
+
+  try {
+    const service = new LeaderboardService()
+
+    for (const channelId of CHANNEL_IDS) {
+      try {
+        console.log(`refreshing leaderboard for ${channelId}...`)
+        await service.refreshLeaderboard(channelId)
+      } catch (err) {
+        console.error('refresh failed:', err)
+        return new Response('internal error', { status: 500 })
+      }
+    }
+
+    return Response.json({ success: true })
+  } catch (err) {
+    console.error('refresh failed:', err)
+    return new Response('internal error', { status: 500 })
+  }
+}

src/env.js

@@ -12,6 +12,7 @@ export const env = createEnv({
         ? z.string()
         : z.string().optional(),
     AUTH_DISCORD_ID: z.string(),
+    CRON_SECRET: z.string(),
     AUTH_DISCORD_SECRET: z.string(),
     DISCORD_BOT_TOKEN: z.string(),
     DATABASE_URL: z.string().url(),
@@ -42,6 +43,7 @@ export const env = createEnv({
     DISCORD_BOT_TOKEN: process.env.DISCORD_BOT_TOKEN,
     REDIS_URL: process.env.REDIS_URL,
     NODE_ENV: process.env.NODE_ENV,
+    CRON_SECRET: process.env.CRON_SECRET,
   },
   /**
    * Run `build` or `dev` with `SKIP_ENV_VALIDATION` to skip env validation. This is especially

src/server/api/routers/history.ts

@@ -1,7 +1,9 @@
 import { createTRPCRouter, publicProcedure } from '@/server/api/trpc'
 import { db } from '@/server/db'
-import { player_games, raw_history } from '@/server/db/schema'
+import { metadata, player_games, raw_history } from '@/server/db/schema'
 import { desc, eq } from 'drizzle-orm'
+import ky from 'ky'
+import { chunk } from 'remeda'
 import { z } from 'zod'
 
 export const history_router = createTRPCRouter({
@@ -12,26 +14,69 @@ export const history_router = createTRPCRouter({
       })
     )
     .query(async ({ ctx, input }) => {
-      const entries = await ctx.db
+      return await ctx.db
         .select()
         .from(player_games)
         .where(eq(player_games.playerId, input.user_id))
         .orderBy(desc(player_games.gameNum))
-      return entries
     }),
   sync: publicProcedure.mutation(async () => {
-    await db.delete(raw_history).execute()
-    await db.delete(player_games).execute()
-    // const chunkedData = chunk(data, 100)
-    // for (const chunk of chunkedData) {
-    //   await insertGameHistory(chunk).catch((e) => {
-    //     console.error(e)
-    //   })
-    // }
-    // return data
+    return syncHistory()
   }),
 })
 
+export async function syncHistory() {
+  const cursor = await db
+    .select()
+    .from(metadata)
+    .where(eq(metadata.key, 'history_cursor'))
+    .limit(1)
+    .then((res) => res[0])
+  console.log('cursor', cursor)
+  const data = await ky
+    .get('https://api.neatqueue.com/api/history/1226193436521267223', {
+      searchParams: {
+        start_game_number: cursor?.value ?? 1,
+      },
+      timeout: 60000,
+    })
+    .json<any>()
+  const matches = await fetch(
+    'https://api.neatqueue.com/api/matches/1226193436521267223'
+  ).then((res) => res.json())
+  const firstGame = Object.keys(matches).sort(
+    (a, b) => Number.parseInt(a) - Number.parseInt(b)
+  )[0]
+  if (!firstGame) {
+    throw new Error('No first game found')
+  }
+
+  await db
+    .insert(metadata)
+    .values({
+      key: 'history_cursor',
+      value: firstGame,
+    })
+    .onConflictDoUpdate({
+      target: metadata.key,
+      set: {
+        key: 'history_cursor',
+        value: firstGame,
+      },
+    })
+  console.log('matches', matches)
+  console.log('firstGame', firstGame)
+  console.log('data', data)
+
+  const chunkedData = chunk(data.data, 100)
+  for (const chunk of chunkedData) {
+    await insertGameHistory(chunk).catch((e) => {
+      console.error(e)
+    })
+  }
+  return data
+}
+
 function processGameEntry(gameId: number, game_num: number, entry: any) {
   const parsedEntry = typeof entry === 'string' ? JSON.parse(entry) : entry
   if (parsedEntry.game === '1v1-attrition') {
@@ -98,14 +143,36 @@ function processGameEntry(gameId: number, game_num: number, entry: any) {
   ]
 }
 export async function insertGameHistory(entries: any[]) {
-  const rawResults = await db
+  const rawResults = await Promise.all(
+    entries.map(async (entry) => {
+      return db
         .insert(raw_history)
-    .values(entries.map((entry) => ({ entry, game_num: entry.game_num })))
+        .values({ entry, game_num: entry.game_num })
         .returning()
+        .onConflictDoUpdate({
+          target: raw_history.game_num,
+          set: {
+            entry,
+          },
+        })
+        .then((res) => res[0])
+    })
+  ).then((res) => res.filter(Boolean))
 
-  const playerGameRows = rawResults.flatMap(({ entry, id, game_num }) => {
+  const playerGameRows = rawResults.flatMap(({ entry, id, game_num }: any) => {
     return processGameEntry(id, game_num, entry)
   })
 
-  await db.insert(player_games).values(playerGameRows).onConflictDoNothing()
+  await Promise.all(
+    playerGameRows.map(async (row) => {
+      return db
+        .insert(player_games)
+        .values(row)
+        .onConflictDoUpdate({
+          target: [player_games.playerId, player_games.gameNum],
+          set: row,
+        })
+        .then((res) => res[0])
+    })
+  )
 }

src/server/db/schema.ts

@@ -1,6 +1,5 @@
 import { relations, sql } from 'drizzle-orm'
 import {
-  boolean,
   index,
   integer,
   json,
@@ -22,8 +21,13 @@ export const raw_history = pgTable(
   },
   (t) => [uniqueIndex('game_num_unique_idx').on(t.game_num)]
 )
-
-export const player_games = pgTable('player_games', {
+export const metadata = pgTable('metadata', {
+  key: text('key').primaryKey().notNull(),
+  value: text('value').notNull(),
+})
+export const player_games = pgTable(
+  'player_games',
+  {
     playerId: text('player_id').notNull(),
     playerName: text('player_name').notNull(),
     gameId: integer('game_id').notNull(),
@@ -36,7 +40,12 @@ export const player_games = pgTable('player_games', {
     opponentName: text('opponent_name').notNull(),
     opponentMmr: real('opponent_mmr').notNull(),
     result: text('result').notNull(),
-})
+  },
+  (t) => [
+    primaryKey({ columns: [t.playerId, t.gameNum] }),
+    uniqueIndex('game_num_per_player_idx').on(t.playerId, t.gameNum),
+  ]
+)
 
 export const users = pgTable('user', (d) => ({
   id: d