Fix for upserts, question pages (WIP), history

docs/graphql.md

@@ -24,6 +24,14 @@ List of all files used for graphql:
 
 `graphql-code-generator` converts those into `*.generated.ts` files which can be imported from the React components.
 
+# Notes on caching
+
+`urql` has both [document caching](https://formidable.com/open-source/urql/docs/basics/document-caching/) and [normalized caching](https://formidable.com/open-source/urql/docs/graphcache/normalized-caching/) (which we don't use yet).
+
+Unfortunately, it's useful only on a page level: since we like server-side rendering, we still have to hit `getServerSideProps` on navigation, even if we have data in cache.
+
+There are some possible workaround for this to make client-side navigation faster, but none of them are trivial to implement; relevant Next.js discussion to follow: https://github.com/vercel/next.js/discussions/19611
+
 # Recipes
 
 **I want to check out what Metaforecast's GraphQL API is capable of**

prisma/migrations/20220425220646_history_relation/migration.sql

@@ -0,0 +1,5 @@
+ALTER TABLE "history" ADD COLUMN     "idref" TEXT;
+
+ALTER TABLE "history" ADD CONSTRAINT "history_idref_fkey" FOREIGN KEY ("idref") REFERENCES "questions"("id") ON DELETE SET NULL ON UPDATE RESTRICT;
+
+UPDATE "history" SET idref = id WHERE id in (SELECT id FROM "questions");

prisma/schema.prisma

@@ -1,5 +1,6 @@
 generator client {
-  provider = "prisma-client-js"
+  provider        = "prisma-client-js"
+  previewFeatures = ["interactiveTransactions"]
 }
 
 generator pothos {
@@ -25,15 +26,17 @@ model Dashboard {
 
 model History {
   id                String
+  idref             String?
+  question          Question? @relation(fields: [idref], references: [id], onDelete: SetNull, onUpdate: Restrict)
   title             String
   url               String
   platform          String
   description       String
   options           Json
-  timestamp         DateTime @db.Timestamp(6)
+  timestamp         DateTime  @db.Timestamp(6)
   qualityindicators Json
   extra             Json
-  pk                Int      @id @default(autoincrement())
+  pk                Int       @id @default(autoincrement())
 
   @@index([id])
   @@map("history")
@@ -75,6 +78,8 @@ model Question {
   extra             Json
 
   onFrontpage FrontpageId?
+  history     History[]
+
   @@map("questions")
 }
 

schema.graphql

@@ -93,6 +93,9 @@ type Query {
 
   """Get a list of questions that are currently on the frontpage"""
   frontpage: [Question!]!
+
+  """Look up a single question by its id"""
+  question(id: ID!): Question!
   questions(after: String, before: String, first: Int, last: Int): QueryQuestionsConnection!
 
   """

src/backend/flow/history/updateHistory.ts

@@ -3,6 +3,9 @@ import { prisma } from "../../database/prisma";
 export async function updateHistory() {
   const questions = await prisma.question.findMany({});
   await prisma.history.createMany({
-    data: questions,
+    data: questions.map((q) => ({
+      ...q,
+      idref: q.id,
+    })),
   });
 }

src/backend/platforms/index.ts

@@ -89,27 +89,69 @@ export const processPlatform = async (platform: Platform) => {
     console.log(`Platform ${platform.name} doesn't have a fetcher, skipping`);
     return;
   }
-  const results = await platform.fetcher();
-  if (results && results.length) {
-    await prisma.$transaction([
-      prisma.question.deleteMany({
-        where: {
-          platform: platform.name,
-        },
-      }),
-      prisma.question.createMany({
-        data: results.map((q) => ({
-          extra: {},
-          timestamp: new Date(),
-          ...q,
-          qualityindicators: q.qualityindicators as object, // fighting typescript
-        })),
-      }),
-    ]);
-    console.log("Done");
-  } else {
+  const fetchedQuestions = await platform.fetcher();
+  if (!fetchedQuestions || !fetchedQuestions.length) {
     console.log(`Platform ${platform.name} didn't return any results`);
+    return;
+  }
+
+  const prepareQuestion = (q: FetchedQuestion): Question => {
+    return {
+      extra: {},
+      timestamp: new Date(),
+      ...q,
+      platform: platform.name,
+      qualityindicators: q.qualityindicators as object, // fighting typescript
+    };
+  };
+
+  const oldQuestions = await prisma.question.findMany({
+    where: {
+      platform: platform.name,
+    },
+  });
+
+  const fetchedIds = fetchedQuestions.map((q) => q.id);
+  const oldIds = oldQuestions.map((q) => q.id);
+
+  const fetchedIdsSet = new Set(fetchedIds);
+  const oldIdsSet = new Set(oldIds);
+
+  const createdQuestions: Question[] = [];
+  const updatedQuestions: Question[] = [];
+  const deletedIds = oldIds.filter((id) => !fetchedIdsSet.has(id));
+
+  for (const q of fetchedQuestions.map((q) => prepareQuestion(q))) {
+    if (oldIdsSet.has(q.id)) {
+      updatedQuestions.push(q);
+    } else {
+      // TODO - check if question has changed for better performance
+      createdQuestions.push(q);
+    }
   }
+
+  await prisma.question.createMany({
+    data: createdQuestions,
+  });
+
+  for (const q of updatedQuestions) {
+    await prisma.question.update({
+      where: { id: q.id },
+      data: q,
+    });
+  }
+
+  await prisma.question.deleteMany({
+    where: {
+      id: {
+        in: deletedIds,
+      },
+    },
+  });
+
+  console.log(
+    `Done, ${deletedIds.length} deleted, ${updatedQuestions.length} updated, ${createdQuestions.length} created`
+  );
 };
 
 export interface PlatformConfig {

src/backend/platforms/xrisk.ts

@@ -15,11 +15,15 @@ export const xrisk: Platform = {
       encoding: "utf-8",
     });
     let results = JSON.parse(fileRaw);
-    results = results.map((item) => ({
-      ...item,
-      id: `${platformName}-${hash(item.title + " | " + item.url)}`, // some titles are non-unique, but title+url pair is always unique
-      platform: platformName,
-    }));
+    results = results.map((item) => {
+      item.extra = item.moreoriginsdata;
+      delete item.moreoriginsdata;
+      return {
+        ...item,
+        id: `${platformName}-${hash(item.title + " | " + item.url)}`, // some titles are non-unique, but title+url pair is always unique
+        platform: platformName,
+      };
+    });
     return results;
   },
 };

src/graphql/schema/questions.ts

@@ -1,3 +1,5 @@
+import { History, Question } from "@prisma/client";
+
 import { prisma } from "../../backend/database/prisma";
 import { platforms, QualityIndicators } from "../../backend/platforms";
 import { builder } from "../builder";
@@ -78,44 +80,67 @@ export const ProbabilityOptionObj = builder
     }),
   });
 
+const QuestionShapeInterface = builder
+  .interfaceRef<Question | History>("QuestionShape")
+  .implement({
+    fields: (t) => ({
+      title: t.exposeString("title"),
+      description: t.exposeString("description"),
+      url: t.exposeString("url", {
+        description:
+          "Non-unique, a very small number of platforms have a page for more than one prediction",
+      }),
+      platform: t.field({
+        type: PlatformObj,
+        resolve: (parent) => parent.platform,
+      }),
+      timestamp: t.field({
+        type: "Date",
+        description: "Timestamp at which metaforecast fetched the question",
+        resolve: (parent) => parent.timestamp,
+      }),
+      qualityIndicators: t.field({
+        type: QualityIndicatorsObj,
+        resolve: (parent) =>
+          parent.qualityindicators as any as QualityIndicators,
+      }),
+      options: t.field({
+        type: [ProbabilityOptionObj],
+        resolve: ({ options }) => {
+          if (!Array.isArray(options)) {
+            return [];
+          }
+          return options as any[];
+        },
+      }),
+    }),
+  });
+
+export const HistoryObj = builder.prismaObject("History", {
+  findUnique: (history) => ({ pk: history.pk }),
+  interfaces: [QuestionShapeInterface],
+  fields: (t) => ({
+    id: t.exposeID("pk", {
+      description: "History items are identified by their integer ids",
+    }),
+    questionId: t.exposeID("id", {
+      description: "Unique string which identifies the question",
+    }),
+  }),
+});
+
 export const QuestionObj = builder.prismaObject("Question", {
   findUnique: (question) => ({ id: question.id }),
+  interfaces: [QuestionShapeInterface],
   fields: (t) => ({
     id: t.exposeID("id", {
       description: "Unique string which identifies the question",
     }),
-    title: t.exposeString("title"),
-    description: t.exposeString("description"),
-    url: t.exposeString("url", {
-      description:
-        "Non-unique, a very small number of platforms have a page for more than one prediction",
-    }),
-    timestamp: t.field({
-      type: "Date",
-      description: "Timestamp at which metaforecast fetched the question",
-      resolve: (parent) => parent.timestamp,
-    }),
-    platform: t.field({
-      type: PlatformObj,
-      resolve: (parent) => parent.platform,
-    }),
-    qualityIndicators: t.field({
-      type: QualityIndicatorsObj,
-      resolve: (parent) => parent.qualityindicators as any as QualityIndicators,
-    }),
-    options: t.field({
-      type: [ProbabilityOptionObj],
-      resolve: ({ options }) => {
-        if (!Array.isArray(options)) {
-          return [];
-        }
-        return options as any[];
-      },
-    }),
     visualization: t.string({
       resolve: (parent) => (parent.extra as any)?.visualization, // used for guesstimate only, see searchGuesstimate.ts
       nullable: true,
     }),
+    history: t.relation("history", {}),
   }),
 });
 
@@ -125,10 +150,26 @@ builder.queryField("questions", (t) =>
       type: "Question",
       cursor: "id",
       maxSize: 1000,
-      resolve: (query, parent, args, context, info) =>
-        prisma.question.findMany({ ...query }),
+      resolve: (query) => prisma.question.findMany({ ...query }),
     },
     {},
     {}
   )
 );
+
+builder.queryField("question", (t) =>
+  t.field({
+    type: QuestionObj,
+    description: "Look up a single question by its id",
+    args: {
+      id: t.arg({ type: "ID", required: true }),
+    },
+    resolve: async (parent, args) => {
+      return await prisma.question.findUnique({
+        where: {
+          id: String(args.id),
+        },
+      });
+    },
+  })
+);

src/graphql/types.generated.ts

@@ -99,6 +99,8 @@ export type Query = {
   dashboard: Dashboard;
   /** Get a list of questions that are currently on the frontpage */
   frontpage: Array<Question>;
+  /** Look up a single question by its id */
+  question: Question;
   questions: QueryQuestionsConnection;
   /** Search for questions; uses Algolia instead of the primary metaforecast database */
   searchQuestions: Array<Question>;
@@ -110,6 +112,11 @@ export type QueryDashboardArgs = {
 };
 
 
+export type QueryQuestionArgs = {
+  id: Scalars['ID'];
+};
+
+
 export type QueryQuestionsArgs = {
   after?: InputMaybe<Scalars['String']>;
   before?: InputMaybe<Scalars['String']>;

src/pages/about.tsx

@@ -1,7 +1,9 @@
+import { NextPage } from "next";
 import React from "react";
 import ReactMarkdown from "react-markdown";
 import gfm from "remark-gfm";
 
+import { Card } from "../web/display/Card";
 import { Layout } from "../web/display/Layout";
 
 const readmeMarkdownText = `# About
@@ -26,16 +28,16 @@ Also note that, whatever other redeeming features they might have, prediction ma
 
 `;
 
-export default function About() {
+const AboutPage: NextPage = () => {
   return (
     <Layout page="about">
-      <div className="px-2 py-2 bg-white rounded-md shadow place-content-stretch flex-grow place-self-center">
-        <ReactMarkdown
-          remarkPlugins={[gfm]}
-          children={readmeMarkdownText}
-          className="m-5"
-        />
-      </div>
+      <Card highlightOnHover={false}>
+        <div className="p-4">
+          <ReactMarkdown remarkPlugins={[gfm]} children={readmeMarkdownText} />
+        </div>
+      </Card>
     </Layout>
   );
-}
+};
+
+export default AboutPage;

src/pages/questions/[id].tsx

@@ -0,0 +1,4 @@
+export {
+  default,
+  getServerSideProps,
+} from "../../web/questions/pages/QuestionPage";