Skip to content

Commit

Permalink
Feat/bulk/all (#2020)
Browse files Browse the repository at this point in the history 
* feat: bulk frontend modal (#1958)

* chore: create new CreateTypeButton for CreateLinkForm

* chore: swap isFile state boolean to createType enum

* chore: add CreateType BULK button

* feat: add BulkUpload form

* chore: fix button disabling check

* chore: add download sample csv functionality

* fix: bugs

* chore: add user actions for bulk create

* fix: misspelling and children nesting

* chore: remove isFile from LinkFormStyles

* chore: try to fix the css

* chore: refactor FileInputField component (#1960)

* chore: refactor FileInputField component

* fix: remove unused styles

* feat: connect bulk csv upload to backend

* fix: tag serialization and deserialization on FormData

* fix: renamed button

* chore: add e2e tests for bulk creation (#2022)

* chore: add e2e tests for bulk creation

* chore: clean up unused code

Co-authored-by: halfwhole <[email protected]>

* chore: add small wait time to stop test flakiness

Co-authored-by: halfwhole <[email protected]>

* feat(bulk-backend): Job and JobItem DB model, repository and service (#2007)

* Feat/link audit to include tags (#1950)

* feat: added tags table and many to many rel to urls

* feat: create tag API to include tags

* feat: create tag API to include tags

* feat: create tag API to include tags

* feat: create tag API to include tags

* feat: create tag API to include tags

* feat: create tag API to include tags

* feat: create tag API to include tags

* feat: create tag API to include tags

* feat: create tag API to include tags

* feat: create tag API to include tags

* feat: create tag API to include tags

* feat: create tag API to include tags

* feat: create tag API to include tags

* feat: create tag API to include tags

* feat: create url API to include tags

* feat: create url API to include tags

* feat: create url API to include tags

* feat: create url API to include tags

* feat: create url API to include tags

* feat: create url API to include tags

* feat: create url API to include tags

* feat: create url API to include tags

* feat: update url API to include tags

* feat: update url API to include tags

* feat: update url API to include tags

* feat: update url API to include tags

* feat: update url API to include tags

* feat: update url API to include tags

* feat: update url API to include tags

* feat: update url API to include tags

* feat: update url API to include tags

* feat: update url API to include tags

* feat: update url API to include tags

* feat: update url API to include tags

* feat: update url API to include tags

* feat: search-link-with-tags

* feat: search-link-with-tags

* feat: search-link-with-tags

* feat: search-link-with-tags

* feat: search-link-with-tags

* feat: search-link-with-tags

* feat: search-link-with-tags

* feat: search-link-with-tags

* feat: link-audit-to-include-tags

* feat: link-audit-to-include-tags

* feat: link-audit-to-include-tags

* feat: link-audit-to-include-tags

* feat: link-audit-to-include-tags

* feat: link-audit-to-include-tags

* feat: link-audit-to-include-tags

* feat: link-audit-to-include-tags

* feat: link-audit-to-include-tags

* feat: link-audit-to-include-tags

* feat: link-audit-to-include-tags

* feat: link-audit-to-include-tags

* feat: link-audit-to-include-tags

* feat: link-audit-to-include-tags

* feat: link-audit-to-include-tags

* feat: link-audit-to-include-tags

* feat: link-audit-to-include-tags

* feat: link-audit-to-include-tags

* feat: link-audit-to-include-tags

* feat: link-audit-to-include-tags

* feat: link-audit-to-include-tags

* feat: link-audit-to-include-tags

* feat: link-audit-to-include-tags

* feat: link-audit-to-include-tags

* feat: link-audit-to-include-tags

* feat: link-audit-to-include-tags

* feat: merge link-audit-to-include-tags with develop

* feat: link-audit-to-include-tags

* feat: search for tags in lower cap (#1973)

* feat: search for tags in lower cap

* feat: search for tags in lower cap

* feat(tagging): add tags to link audit history (#1974)

* chore(datadog): refactor custom metric names (#1965)

* feat(tagging): add frontend tagging on create new link form (#1919)

* feat(tagging): add frontend tagging on create new link form

* chore: refactor autocomplete and address comments

* chore: refactor text field height and add disable submit checks

* chore: refactor max tag length in helper error message

* chore(tagging): refactor max number of tags per link

* chore(tagging): update tag validation tests for underscores

* feat(tagging): add dropdown for tags on user page search bar (#1934)

* feat(tagging): add dropdown for tags on user page search bar

* feat(tagging): add link icon on search dropdown

* feat(tagging): refactor link icon and add tag icon on search dropdown

* feat(tagging): revert link icon to original size

* build(deps): bump winston from 3.3.3 to 3.8.1 (#1889)

* build(deps): bump express-fileupload from 1.2.1 to 1.4.0 (#1890)

* feat(tagging): add frontend API integrations on create new link form (#1971)

* feat(tagging): add frontend tagging on create new link form

* chore: refactor autocomplete and address comments

* chore: refactor text field height and add disable submit checks

* chore: refactor max tag length in helper error message

* chore(tagging): refactor max number of tags per link

* chore(tagging): update tag validation tests for underscores

* feat(tagging): add frontend API integrations for create new link form

* chore(tagging): update tag validation tests for underscores

* chore(tagging): refactor tags to use redux instead of useState

* Feat/add tag to link audit frontend (#1975)

* feat: added TagList component by using mui/Chip to show tags in Link Audit

* feat: added TagList component by using mui/Chip to show tags in Link Audit

* feat: added TagList component by using mui/Chip to show tags in Link Audit

* feat(tagging): add editing of link tags (#1976)

* feat(tagging): add frontend tagging on create new link form

* chore: refactor autocomplete and address comments

* chore: refactor text field height and add disable submit checks

* chore: refactor max tag length in helper error message

* chore(tagging): refactor max number of tags per link

* chore(tagging): update tag validation tests for underscores

* feat(tagging): add frontend API integrations for create new link form

* chore(tagging): update tag validation tests for underscores

* chore(tagging): refactor tags to use redux instead of useState

* feat(tagging): add editing of link tags

* feat(tagging): use manual save button for editing link tags

* fix: return successful response when no URLs are found (#1979)

* fix(tagging): reset tags state after link creation (#1982)

* feat(tagging): add tags to links on user page (#1972)

* feat(tagging): add tags to links on user page

* feat(tagging): apply search when clicking on tags

* chore(tagging): remove duplicate identifier for tags

* fix: revise findUrlsForUser to return urls in correct order (#1981)

* fix: revise findUrlsForUser to return urls in correct order

* fix: revise findUrlsForUser to return urls in correct order

* fix: revise findUrlsForUser to return urls in correct order

* fix: revise findUrlsForUser to return urls in correct order

* fix: revise findUrlsForUser to return urls in correct order

* fix: revise findUrlsForUser to return urls in correct order

* fix: revise findUrlsForUser to return urls in correct order

* fix: revise findUrlsForUser to return urls in correct order

* fix: revise findUrlsForUser to return urls in correct order

* fix: fix urlMapper to correctly check for empty tagStrings before parsing (#1989)

* chore: remove disallowed file extensions (#1985)

* Fix/link tagging for file upload (#1986)

* fix: add tags when creating new link from file

* chore: update sentry message

* fix: wrap backend errors in json messages (#1991)

* fix(tagging): serialize single tags for file uploads (#1990)

* fix(tagging): serialize single tags for file uploads

* chore(tagging): add comment for preprocessing file tags

* feat: add job and job item models

* feat: add jobmanagementservice

* chore: fix conflicts

* chore: rename JobStatusEnum to JobItemStatusEnum

* chore: remove JobItemStatusEnum - Ready

* chore: replaced isSuccess to getJobStatus in JobManagementService

Co-authored-by: thanhdatle <[email protected]>
Co-authored-by: halfwhole <[email protected]>

* chore: add error message for invalid date string and csv header

* Feat/bulk/frontend/status bar (#2040)

* chore: add icons

* feat: add StatusBarComponent

* chore: fix alignment

* chore: add actions and reducers

* chore: fix JSX linting error with eslint

* chore: attach actions and reducers to StatusBar component

* chore: display QR code inprogress after bulk upload (#2041)

* fix: combine class styles

* chore: add docs inline for disabling eslint no-undef

* Feat/bulk/frontend/status bar button (#2047)

* chore: refactor existing DownloadButton to separate component from logic

* chore: add CloseButton as a reusable component

* chore: add DownloadBulkButton to StatusBar

* chore: styling

* chore: rename icon

* Feat/bulk/all lambda (#2033)

* feat: add lambda deployment and sample code

* feat: add qrcode and zip lambda code

* feat: add createCsv, uploadToS3 and sendsqsMessage logic

* feat: update handler logic

* feat: stream files to s3 bulk-qrcode-generation

* chore: add assets

* chore: add sns publish method

* chore: refactor code

* chore: temp remove unnecessary ci build commadns

* chore: fix qrcode url

* chore: import services instead of functions

* Revert "chore: temp remove unnecessary ci build commadns"

This reverts commit f4474a19e25a82aa0205fdc5bdb7c7ef8562824a.

* chore: set memorySize, ephemeralStorageSize, timeout on serverless

* fix: clean up dir on err

* chore: use archiver directory false

* chore: remove streamToS3 from exports

* chore: set domain in env variables

Co-authored-by: Alexis <[email protected]>

* Feat/bulk/all cleanup unused (#2082)

* chore: remove JobItemType from JobItem

* chore: remove JobItemStatusEnum Ready

* chore: refactor Job module to group files and remove mappers

Co-authored-by: Jim Engay <[email protected]>

* Feat/bulk/all create job (#2083)

* feat: add sqs client and service

* feat: create job and job items and sendSqsmessage in bulk controller

* chore: move qrCode job creation into JobController

* chore: generalize sqs message sending

* chore: add sqs timeout

* chore: fix missing region

* chore: create qrBatchSize and sqsRegion env variable

* chore: fix error logging and type

* chore: fix bulk tests

* chore: fix lambda params

* chore: fix env variable documentation

Co-authored-by: Jim Engay <[email protected]>

* Feat/bulk/all update job (#2085)

* feat: add sqs client and service

* feat: create job and job items and sendSqsmessage in bulk controller

* chore: move qrCode job creation into JobController

* chore: generalize sqs message sending

* chore: add sqs timeout

* chore: fix missing region

* chore: create qrBatchSize and sqsRegion env variable

* chore: fix error logging and type

* chore: fix bulk tests

* chore: fix lambda params

* chore: add JobItemId to job_items model for easy update

* chore: refactor logic to update jobItemStatus

* feat: expose endpoint to update jobItem

* chore: use env var for lambda hash value

* chore: use http post request to send completion callback

* chore: fix validation

* chore: add tests

* chore: fix error catching for lambda

* chore: add status to job table

* chore: add logic to update job after job item callback

* chore: fix tests

* chore: fix tests

* fix: call next to pass control

* fix: typo in docs

Co-authored-by: halfwhole <[email protected]>

* chore: refactor job status computation logic

Co-authored-by: halfwhole <[email protected]>

* chore: fix lint errors after merging suggestions

* chore: attach only jobId to req body in updateJobItem

* chore: change enum from Failed to Failure

* chore: add unique constraint on jobItemId

Co-authored-by: Jim Engay <[email protected]>
Co-authored-by: halfwhole <[email protected]>

* Feat/bulk/all get job server (#2092)

* feat: add sqs client and service

* feat: create job and job items and sendSqsmessage in bulk controller

* chore: move qrCode job creation into JobController

* chore: generalize sqs message sending

* chore: add sqs timeout

* chore: fix missing region

* chore: create qrBatchSize and sqsRegion env variable

* chore: fix error logging and type

* chore: fix bulk tests

* chore: fix lambda params

* chore: add JobItemId to job_items model for easy update

* chore: refactor logic to update jobItemStatus

* feat: expose endpoint to update jobItem

* chore: use env var for lambda hash value

* chore: use http post request to send completion callback

* chore: fix validation

* chore: add tests

* chore: fix error catching for lambda

* chore: add status to job table

* chore: add logic to update job after job item callback

* chore: fix tests

* chore: fix tests

* fix: call next to pass control

* fix: typo in docs

Co-authored-by: halfwhole <[email protected]>

* chore: refactor job status computation logic

Co-authored-by: halfwhole <[email protected]>

* chore: fix lint errors after merging suggestions

* chore: attach only jobId to req body in updateJobItem

* chore: change enum from Failed to Failure

* fix: add env variable to feature flag job creation

* chore: add repository methods to retrieve jobs

* chore: add service methods for long polling and retrieving job information

* chore: add endpoints for users to retrieve their latest job and updated job status

* chore: add tests

* fix: add env variables to docker-compose

* chore: add env variable for bulk download bucket

* chore: fix tests

* chore: fix typos in test

* chore: fix tests

* chore: change job status endpoint to get

* chore: rename jobItemIds to jobItemUrls

* chore: update error statuses

* chore: fix lint errors

Co-authored-by: Jim Engay <[email protected]>
Co-authored-by: halfwhole <[email protected]>

* Feat/bulk/all get job client (#2093)

* feat: add sqs client and service

* feat: create job and job items and sendSqsmessage in bulk controller

* chore: move qrCode job creation into JobController

* chore: generalize sqs message sending

* chore: add sqs timeout

* chore: fix missing region

* chore: create qrBatchSize and sqsRegion env variable

* chore: fix error logging and type

* chore: fix bulk tests

* chore: fix lambda params

* chore: add JobItemId to job_items model for easy update

* chore: refactor logic to update jobItemStatus

* feat: expose endpoint to update jobItem

* chore: use env var for lambda hash value

* chore: use http post request to send completion callback

* chore: fix validation

* chore: add tests

* chore: fix error catching for lambda

* chore: add status to job table

* chore: add logic to update job after job item callback

* chore: fix tests

* chore: fix tests

* fix: call next to pass control

* fix: add env variable to feature flag job creation

* chore: add repository methods to retrieve jobs

* chore: add service methods for long polling and retrieving job information

* chore: add endpoints for users to retrieve their latest job and updated job status

* chore: add tests

* fix: add env variables to docker-compose

* chore: add env variable for bulk download bucket

* fix: env variable naming in docker-compose

* chore: fix tests

* chore: fix tests

* fix: status bar should appear even if no urls

* chore: pass download urls to Download QR code button

* chore: add callback variable to actions and reducers

* chore: remove redundant bulkQRCodesStarted action

* chore: add client-side logic to retrieve user job

* chore: add component logic for status bar

* chore: fix end to end tests

* fix: uncomment sqs.sendService function

* chore: update api call on client for job update

Co-authored-by: Jim Engay <[email protected]>

* Feat/bulk/all email send (#2099)

* chore: make sender domain optional

* chore: add email sending for job completion

* chore: fix local mailDev logging for non-OTP emails

* chore: fix types

* chore: fix tests

* Fix/bulk/all errors (#2102)

* chore: add dogstatsd variables

* fix: add promise to papaparse and refactor bulk validation

* chore: fix bulk upload error styling and handling

* chore: fix tests

* chore: fix types

* chore: add enums to shared folder

* chore: add email sending logic into email service

* chore: fix copy (#2103)

* chore: fix copy

* chore: update copy

Co-authored-by: halfwhole <[email protected]>

Co-authored-by: halfwhole <[email protected]>

* Feat/bulk/all api guard (#2104)

* chore: refactor guard to use api key access

* chore: remove unused env variables and files

* chore: add tests

* chore: remove admin default email

* chore: fix email copy (#2107)

* chore: get header from shared folder

* chore: create shared types for bulk download file names

* chore: remove duplicate REPLICA_URI

* chore: add unauthorized error message

Co-authored-by: Jim Engay <[email protected]>
Co-authored-by: halfwhole <[email protected]>
Co-authored-by: Jim Vincent Andes Engay <[email protected]>
Co-authored-by: thanhdatle <[email protected]>
  • Loading branch information
@gweiying @halfwhole
@jimvae @thanhdatle
5 people committed Dec 6, 2022
1 parent 5c6bae2 commit f3944d2
Show file tree
Hide file tree
Showing 82 changed files with 4,327 additions and 476 deletions.
12 changes: 11 additions & 1 deletion .eslintrc.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,5 +51,15 @@
"extensions": [".jsx", ".js", ".tsx", ".ts", ".scss", ".css", ".png", ".svg"]
}
}
}
},
"overrides": [
{
"files": ["*.ts", "*.tsx"],
"rules": {
// Disabling `no-undef` lint rule in favor of Typescript checking
// https://github.com/typescript-eslint/typescript-eslint/blob/main/docs/linting/TROUBLESHOOTING.md#i-get-errors-from-the-no-undef-rule-about-global-variables-not-being-defined-even-though-there-are-no-typescript-errors
"no-undef": "off"
}
}
]
}
2 changes: 1 addition & 1 deletion Dockerfile.maildev-logging
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
FROM maildev/maildev:1.1.0
WORKDIR /usr/src/app
RUN echo "maildev.on('new', ({ html }) => console.log('Login OTP: ' + html.match(/\d{6}/)[0]))" >> bin/maildev
RUN echo "maildev.on('new', ({ html }) => { if(html.includes('OTP')) { console.log('Login OTP: ' + html.match(/\d{6}/)[0]) } else { console.log(html) } } )" >> bin/maildev
11 changes: 10 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,9 +133,18 @@ After these have been set up, set the environment variables according to the tab
|COOKIE_MAX_AGE|Yes|Session duration of cookie|
|BULK_UPLOAD_MAX_NUM|No|Maximum number of links that can be bulk uploaded at once. Defaults to 1000|
|BULK_UPLOAD_RANDOM_STR_LENGTH|No|String length of randomly generated shortUrl in bulk upload. Defaults to 8|
|API_LINK_RANDOM_STR_LENGTH|No|String length of randomly generated shortUrl in API created links. Defaults to 8|
|BULK_QR_CODE_BATCH_SIZE|No|Maximum batch size of QR codes to generate in a single Lambda run. Defaults to 1000|
|BULK_QR_CODE_BUCKET_URL|No|Link to download QR codes from|
|ACTIVATE_BULK_QR_CODE_GENERATION|No|Whether to start Lambda for bulk QR code generation or not. Defaults to false|
|REPLICA_URI|Yes|The postgres connection string, e.g. `postgres://postgres:postgres@postgres:5432/postgres`|
|SQS_BULK_QRCODE_GENERATE_START_URL|No|The SQS queue for starting QR code bulk generation Lambda|
|SQS_TIMEOUT|No|Duration of time in ms for sending to SQS queue before timeout. Defaults to 10000ms (10s)|
|SQS_REGION|No|AWS Region of SQS queue for starting QR code bulk generation Lambda|
|JOB_POLL_ATTEMPTS|No|Number of attempts for long polling of job status before timeout of 408 is returned. Defaults to 12|
|JOB_POLL_INTERVAL|No|Interval of time between attempts for long polling of job status in ms. Defaults to 5000ms (5s)|
|API_LINK_RANDOM_STR_LENGTH|No|String length of randomly generated shortUrl in API created links. Defaults to 8|
|FF_EXTERNAL_API|No|Boolean, feature flag for enabling the external API. Defaults to false|
|ADMIN_API_EMAIL|No|Email with admin API access. Defaults to none.|

#### Serverless functions for link migration

Expand Down
9 changes: 9 additions & 0 deletions docker-compose.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,9 +52,18 @@ services:
- ACTIVATE_POSTMAN_FALLBACK=
- BULK_UPLOAD_MAX_NUM=1000
- BULK_UPLOAD_RANDOM_STR_LENGTH=8
- SQS_BULK_QRCODE_GENERATE_START_URL=
- SQS_TIMEOUT=10000
- SQS_REGION=
- BULK_QR_CODE_BATCH_SIZE=1000
- ACTIVATE_BULK_QR_CODE_GENERATION=false
- JOB_POLL_ATTEMPTS=12
- JOB_POLL_INTERVAL=5000
- BULK_QR_CODE_BUCKET_URL=
- API_LINK_RANDOM_STR_LENGTH=8
- API_KEY_SALT=$$2b$$10$$9rBKuE4Gb5ravnvP4xjoPu
- FF_EXTERNAL_API=true
- ADMIN_API_EMAIL=
volumes:
- ./public:/usr/src/gogovsg/public
- ./src:/usr/src/gogovsg/src
Expand Down
178 changes: 178 additions & 0 deletions package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
"@sentry/webpack-plugin": "^1.15.1",
"@types/express-rate-limit": "^5.1.3",
"@types/papaparse": "^5.3.5",
"archiver": "^5.3.1",
"aws-sdk": "^2.1101.0",
"babel-polyfill": "^6.26.0",
"bcrypt": "^5.1.0",
Expand Down
13 changes: 12 additions & 1 deletion serverless.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,5 +46,16 @@ functions:
- 'src/server/serverless/capture-ses-events/**'
- 'package-lock.json'
- 'package.json'

bulk-qrcode-generation:
handler: src/server/serverless/bulk-qrcode-generation/index.handler
memorySize: 2048 # 2048 MB
ephemeralStorageSize: 512 # 512 MB
timeout: 120 # 120 seconds
description: Bulk generates QR codes in svg and png from shortUrl-longUrl mappings
package:
patterns:
- '!./**'
- 'src/server/serverless/bulk-qrcode-generation/**'
- 'package-lock.json'
- 'package.json'

33 changes: 23 additions & 10 deletions src/client/app/util/download.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,27 @@ import useIsIE from '../components/BaseLayout/util/ie'
import { GAEvent } from './ga'
import { UrlTableConfig } from '../../user/reducers/types'
import queryObjFromTableConfig from '../helpers/urlQueryHelper'
import { BULK_UPLOAD_HEADER } from '../../../shared/constants'

export const downloadCsv = (csvString: string, filename: string) => {
const blob = new Blob([csvString], {
type: 'text/csv;charset=utf-8',
})

if (useIsIE()) {
navigator.msSaveBlob(blob, filename)
} else {
saveAs(blob, filename)
}
}

export const downloadSampleBulkCsv = () => {
const headers = BULK_UPLOAD_HEADER
const body = ['https://www.link1.com', 'https://www.link2.com']
const content = [headers, ...body].join('\r\n')
downloadCsv(content, 'sample_bulk.csv')
GAEvent('modal page', 'downloaded bulk sample', 'successful')
}

export const downloadUrls = async (tableConfig: UrlTableConfig) => {
const urlsArr = []
Expand Down Expand Up @@ -59,22 +80,14 @@ export const downloadUrls = async (tableConfig: UrlTableConfig) => {
rootActions.setErrorMessage('Error downloading urls.')
return null
}

const blob = new Blob([urlsArr.join('')], {
type: 'text/csv;charset=utf-8',
})

if (useIsIE()) {
navigator.msSaveBlob(blob, 'urls.csv')
} else {
saveAs(blob, 'urls.csv')
}
downloadCsv(urlsArr.join(''), 'urls.csv')

// Google Analytics: Download links button events
GAEvent('user page', 'download links button', 'successful')
return null
}

export default {
downloadSampleBulkCsv,
downloadUrls,
}
Loading

0 comments on commit f3944d2

Please sign in to comment.