Skip to content

Commit

Permalink
Fix errors
Browse files Browse the repository at this point in the history
  • Loading branch information
@ruchernchong
ruchernchong committed May 3, 2024
1 parent 6095a4f commit fcbc4af
Show file tree
Hide file tree
Showing 4 changed files with 192 additions and 153 deletions.
40 changes: 20 additions & 20 deletions src/config/db.ts
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
import * as mongodb from "mongodb";

const MongoClient = mongodb.MongoClient;

let cachedDb: mongodb.Db | null = null;

const connectToDatabase = async (): Promise<mongodb.Db> => {
if (cachedDb) {
return cachedDb;
}

const client = await MongoClient.connect(process.env.MONGODB_URI);
cachedDb = await client.db("main");

return cachedDb;
};

const db = await connectToDatabase();

export default db;
// import * as mongodb from "mongodb";
//
// const MongoClient = mongodb.MongoClient;
//
// let cachedDb: mongodb.Db | null = null;
//
// const connectToDatabase = async (): Promise<mongodb.Db> => {
// if (cachedDb) {
// return cachedDb;
// }
//
// const client = await MongoClient.connect(process.env.MONGODB_URI);
// cachedDb = await client.db("main");
//
// return cachedDb;
// };
//
// const db = await connectToDatabase();
//
// export default db;
294 changes: 162 additions & 132 deletions src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ import { Hono } from "hono";
import { Document, WithId } from "mongodb";
import { format, subMonths } from "date-fns";
import * as d3 from "d3";
import fs from "fs/promises";
import db from "./config/db";
// import fs from "fs/promises";
// import db from "./config/db";
import { downloadFile } from "./utils/downloadFile";
import { extractZipFile } from "./utils/extractZipFile";
import { Car, COEResult, FUEL_TYPE, UpdateParams } from "./types";
Expand All @@ -21,10 +21,40 @@ const getCarsByFuelType = async (
month: month ?? { $gte: trailingTwelveMonths },
};

const cars: WithId<Car>[] = await db
.collection<Car>("cars")
.find(filter)
.toArray();
// const cars: WithId<Car>[] = await db
// .collection<Car>("cars")
// .find(filter)
// .toArray();

const cars = [
{
_id: { $oid: "65f3fb10fa9ed39b7d6e73d5" },
month: "2024-02",
make: "AUDI",
importer_type: "AMD",
fuel_type: "Electric",
vehicle_type: "Hatchback",
number: "1",
},
{
_id: { $oid: "65f3fb10fa9ed39b7d6e73d9" },
month: "2024-02",
make: "B.M.W.",
importer_type: "AMD",
fuel_type: "Electric",
vehicle_type: "Hatchback",
number: "",
},
{
_id: { $oid: "65f3fb10fa9ed39b7d6e73dc" },
month: "2024-02",
make: "BYD",
importer_type: "AMD",
fuel_type: "Electric",
vehicle_type: "Hatchback",
number: "6",
},
];

return cars.reduce(
(result: WithId<Car>[], { _id, month, make, fuel_type, number }) => {
Expand All @@ -50,75 +80,75 @@ const getCarsByFuelType = async (
);
};

const getLatestMonth = async (): Promise<string> => {
const months = await db.collection<COEResult>("coe").distinct("month");
return months[months.length - 1];
};

export const getCOEResultByMonth = async (
month?: string,
): Promise<WithId<COEResult>[]> => {
const selectedMonth = month || (await getLatestMonth());
return db
.collection<COEResult>("coe")
.find({ month: selectedMonth })
.sort({ bidding_no: 1, vehicle_class: 1 })
.toArray();
};

const EXTRACT_PATH: string = "/tmp";

export const update = async ({
collectionName,
zipFileName,
zipUrl,
keyFields,
}: UpdateParams): Promise<{ message: string }> => {
const collection = db.collection<Document>(collectionName);

try {
const zipFilePath = `${EXTRACT_PATH}/${zipFileName}`;
await downloadFile({ url: zipUrl, destination: zipFilePath });

const extractedFileName = await extractZipFile(zipFilePath, EXTRACT_PATH);
const destinationPath = `${EXTRACT_PATH}/${extractedFileName}`;
console.log(`Destination path:`, destinationPath);

const csvData = await fs.readFile(destinationPath, "utf-8");
const parsedData = d3.csvParse(csvData);

const existingData: WithId<Document>[] = await collection.find().toArray();

const createUniqueKey = <T extends object>(
item: T,
keyFields: Array<keyof T>,
): string =>
keyFields
.filter((field) => item[field])
.map((field) => item[field])
.join("-");

const existingDataMap: Map<string, WithId<Document>> = new Map(
existingData.map((item) => [createUniqueKey(item, keyFields), item]),
);
const newDataToInsert = parsedData.filter(
(newItem) => !existingDataMap.has(createUniqueKey(newItem, keyFields)),
);

let message: string;
if (newDataToInsert.length > 0) {
const result = await collection.insertMany(newDataToInsert);
message = `${result.insertedCount} document(s) inserted`;
} else {
message = `No new data to insert. The provided data matches the existing records.`;
}

return { message };
} catch (error) {
console.error(`An error has occurred:`, error);
throw error;
}
};
// const getLatestMonth = async (): Promise<string> => {
// const months = await db.collection<COEResult>("coe").distinct("month");
// return months[months.length - 1];
// };

// export const getCOEResultByMonth = async (
// month?: string,
// ): Promise<WithId<COEResult>[]> => {
// const selectedMonth = month || (await getLatestMonth());
// return db
// .collection<COEResult>("coe")
// .find({ month: selectedMonth })
// .sort({ bidding_no: 1, vehicle_class: 1 })
// .toArray();
// };

// const EXTRACT_PATH: string = "/tmp";

// export const update = async ({
// collectionName,
// zipFileName,
// zipUrl,
// keyFields,
// }: UpdateParams): Promise<{ message: string }> => {
// const collection = db.collection<Document>(collectionName);
//
// try {
// const zipFilePath = `${EXTRACT_PATH}/${zipFileName}`;
// await downloadFile({ url: zipUrl, destination: zipFilePath });
//
// const extractedFileName = await extractZipFile(zipFilePath, EXTRACT_PATH);
// const destinationPath = `${EXTRACT_PATH}/${extractedFileName}`;
// console.log(`Destination path:`, destinationPath);
//
// const csvData = await fs.readFile(destinationPath, "utf-8");
// const parsedData = d3.csvParse(csvData);
//
// const existingData: WithId<Document>[] = await collection.find().toArray();
//
// const createUniqueKey = <T extends object>(
// item: T,
// keyFields: Array<keyof T>,
// ): string =>
// keyFields
// .filter((field) => item[field])
// .map((field) => item[field])
// .join("-");
//
// const existingDataMap: Map<string, WithId<Document>> = new Map(
// existingData.map((item) => [createUniqueKey(item, keyFields), item]),
// );
// const newDataToInsert = parsedData.filter(
// (newItem) => !existingDataMap.has(createUniqueKey(newItem, keyFields)),
// );
//
// let message: string;
// if (newDataToInsert.length > 0) {
// const result = await collection.insertMany(newDataToInsert);
// message = `${result.insertedCount} document(s) inserted`;
// } else {
// message = `No new data to insert. The provided data matches the existing records.`;
// }
//
// return { message };
// } catch (error) {
// console.error(`An error has occurred:`, error);
// throw error;
// }
// };

app.get("/", async (c) => {
const month = c.req.query("month");
Expand Down Expand Up @@ -150,62 +180,62 @@ app.get("/petrol", async (c) => {
return c.json(cars);
});

app.get("/coe", (c) => {
return c.json(db.collection<COEResult>("coe").find().toArray());
});

app.get("/coe/latest", async (c) => {
const month = c.req.query("month");
const result: WithId<COEResult>[] = await getCOEResultByMonth(month);
return c.json(result);
});

app.get("/updater/cars", async (c) => {
const COLLECTION_NAME: string = "cars";
const ZIP_FILE_NAME: string = "Monthly New Registration of Cars by Make.zip";
const ZIP_URL: string = `https://datamall.lta.gov.sg/content/dam/datamall/datasets/Facts_Figures/Vehicle Registration/${ZIP_FILE_NAME}`;

const { message } = await update({
collectionName: COLLECTION_NAME,
zipFileName: ZIP_FILE_NAME,
zipUrl: ZIP_URL,
keyFields: ["month"],
});

console.log(`Message:`, message);

return c.json({
status: 200,
collection: COLLECTION_NAME,
message,
timestamp: new Date().toISOString(),
});
});

app.get("/updater/coe", async (c) => {
const COLLECTION_NAME: string = "coe";
const ZIP_FILE_NAME: string = "COE Bidding Results.zip";
const ZIP_URL: string = `https://datamall.lta.gov.sg/content/dam/datamall/datasets/Facts_Figures/Vehicle Registration/${ZIP_FILE_NAME}`;

const { message } = await update({
collectionName: COLLECTION_NAME,
zipFileName: ZIP_FILE_NAME,
zipUrl: ZIP_URL,
keyFields: ["month", "bidding_no"],
});

console.log(`Message:`, message);

return c.json({
status: 200,
collection: COLLECTION_NAME,
message,
timestamp: new Date().toISOString(),
});
});

app.get("/vehicle-make", (c) => {
return c.json(db.collection<Car>("cars").distinct("make"));
});
// app.get("/coe", (c) => {
// return c.json(db.collection<COEResult>("coe").find().toArray());
// });
//
// app.get("/coe/latest", async (c) => {
// const month = c.req.query("month");
// const result: WithId<COEResult>[] = await getCOEResultByMonth(month);
// return c.json(result);
// });

// app.get("/updater/cars", async (c) => {
// const COLLECTION_NAME: string = "cars";
// const ZIP_FILE_NAME: string = "Monthly New Registration of Cars by Make.zip";
// const ZIP_URL: string = `https://datamall.lta.gov.sg/content/dam/datamall/datasets/Facts_Figures/Vehicle Registration/${ZIP_FILE_NAME}`;
//
// const { message } = await update({
// collectionName: COLLECTION_NAME,
// zipFileName: ZIP_FILE_NAME,
// zipUrl: ZIP_URL,
// keyFields: ["month"],
// });
//
// console.log(`Message:`, message);
//
// return c.json({
// status: 200,
// collection: COLLECTION_NAME,
// message,
// timestamp: new Date().toISOString(),
// });
// });
//
// app.get("/updater/coe", async (c) => {
// const COLLECTION_NAME: string = "coe";
// const ZIP_FILE_NAME: string = "COE Bidding Results.zip";
// const ZIP_URL: string = `https://datamall.lta.gov.sg/content/dam/datamall/datasets/Facts_Figures/Vehicle Registration/${ZIP_FILE_NAME}`;
//
// const { message } = await update({
// collectionName: COLLECTION_NAME,
// zipFileName: ZIP_FILE_NAME,
// zipUrl: ZIP_URL,
// keyFields: ["month", "bidding_no"],
// });
//
// console.log(`Message:`, message);
//
// return c.json({
// status: 200,
// collection: COLLECTION_NAME,
// message,
// timestamp: new Date().toISOString(),
// });
// });

// app.get("/vehicle-make", (c) => {
// return c.json(db.collection<Car>("cars").distinct("make"));
// });

export default app;
7 changes: 7 additions & 0 deletions sst-env.d.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
/* tslint:disable *//* eslint-disable */import "sst"
declare module "sst" {
export interface Resource {
Database: import("@cloudflare/workers-types").D1Database
}
}
export {}
4 changes: 3 additions & 1 deletion sst.config.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,11 @@ export default $config({
};
},
async run() {
const database = new sst.cloudflare.D1("Database");
const hono = new sst.cloudflare.Worker("Api", {
url: true,
handler: "src/index.ts",
link: [database],
url: true,
});

return {
Expand Down

0 comments on commit fcbc4af

Please sign in to comment.