Skip to content

Commit

Permalink
Merge pull request #2112 from statisticsnorway/MIMIR-854_push-rss-sta…
Browse files Browse the repository at this point in the history 
…tkal-service

Mimir 854 push rss statkal service
  • Loading branch information
@ssb-cgn
ssb-cgn authored Oct 2, 2023
2 parents 913ba3f + fdde697 commit f092de0
Show file tree
Hide file tree
Showing 12 changed files with 345 additions and 20 deletions.
63 changes: 52 additions & 11 deletions src/main/resources/lib/ssb/cron/cron.ts
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
/* eslint-disable complexity */
import { createUser, findUsers } from '/lib/xp/auth'
import type { Content } from '/lib/xp/content'
import { run, type ContextParams } from '/lib/xp/context'
Expand Down Expand Up @@ -233,18 +234,8 @@ export function setupCronJobs(): void {
context: cronContext,
})

// Task
const testTaskCron: string =
app.config && app.config['ssb.cron.testTask'] ? app.config['ssb.cron.testTask'] : '0 08 * * *'
const datasetPublishCron: string =
app.config && app.config['ssb.cron.publishDataset'] ? app.config['ssb.cron.publishDataset'] : '50 05 * * *'
const updateMimirMockReleaseCron: string =
app.config && app.config['ssb.cron.updateMimirReleasedMock']
? app.config['ssb.cron.updateMimirReleasedMock']
: '01 8 * * *'
const updateCalculatorCron: string =
app.config && app.config['ssb.cron.updateCalculator'] ? app.config['ssb.cron.updateCalculator'] : '01 8 * * *'
const timezone: string = app.config && app.config['ssb.cron.timezone'] ? app.config['ssb.cron.timezone'] : 'UTC'

// Use feature-toggling to switch to lib-sheduler when testet in QA
if (!newPublishJobEnabled) {
Expand All @@ -260,7 +251,10 @@ export function setupCronJobs(): void {
log.info('Run new dailyPublishJob lib-scheduler')
}

// Task
if (isMaster()) {
const timezone: string =
app.config && app.config['ssb.cron.timezone'] ? app.config['ssb.cron.timezone'] : 'Europe/Oslo'
// publish dataset sheduler job
run(cronContext, () => {
const jobExists = !!getScheduledJob({
Expand Down Expand Up @@ -298,6 +292,8 @@ export function setupCronJobs(): void {

// Test sheduler task
run(cronContext, () => {
const testTaskCron: string =
app.config && app.config['ssb.cron.testTask'] ? app.config['ssb.cron.testTask'] : '0 10 * * *'
const jobExists = !!getScheduledJob({
name: 'testTask',
})
Expand Down Expand Up @@ -330,6 +326,8 @@ export function setupCronJobs(): void {

// Update calculators
run(cronContext, () => {
const updateCalculatorCron: string =
app.config && app.config['ssb.cron.updateCalculator'] ? app.config['ssb.cron.updateCalculator'] : '01 8 * * *'
const jobExists = !!getScheduledJob({
name: 'updateCalculator',
})
Expand All @@ -338,6 +336,9 @@ export function setupCronJobs(): void {
name: 'updateCalculator',
editor: (job) => {
job.schedule.value = updateCalculatorCron
if (job.schedule.type === 'CRON') {
job.schedule.timeZone = timezone
}
return job
},
})
Expand All @@ -351,7 +352,7 @@ export function setupCronJobs(): void {
schedule: {
type: 'CRON',
value: updateCalculatorCron,
timeZone: 'Europe/Oslo',
timeZone: timezone,
},
})
}
Expand All @@ -360,6 +361,10 @@ export function setupCronJobs(): void {
// Update next release Mimir QA
if (app.config && app.config['ssb.mock.enable'] === 'true') {
run(cronContext, () => {
const updateMimirMockReleaseCron: string =
app.config && app.config['ssb.cron.updateMimirReleasedMock']
? app.config['ssb.cron.updateMimirReleasedMock']
: '01 8 * * *'
const jobExists = !!getScheduledJob({
name: 'updateMimirMockRelease',
})
Expand Down Expand Up @@ -387,6 +392,42 @@ export function setupCronJobs(): void {
}
})
}

// Push Rss Statkal
const pushRssStatkalEnabled: boolean = isEnabled('push-rss-statkal', false, 'ssb')
run(cronContext, () => {
const pushRssStatkalCron: string =
app.config && app.config['ssb.cron.pushRssStatkal'] ? app.config['ssb.cron.pushRssStatkal'] : '10 08 * * *'
const jobExists = !!getScheduledJob({
name: 'pushRssStatkal',
})
if (jobExists) {
modify({
name: 'pushRssStatkal',
editor: (job) => {
job.enabled = pushRssStatkalEnabled
job.schedule.value = pushRssStatkalCron
if (job.schedule.type === 'CRON') {
job.schedule.timeZone = timezone
}
return job
},
})
} else {
create({
name: 'pushRssStatkal',
descriptor: `${app.name}:pushRssStatkal`,
description: 'Push kommende publiseringer til rss/statkal',
user: `user:system:cronjob`,
enabled: pushRssStatkalEnabled,
schedule: {
type: 'CRON',
value: pushRssStatkalCron,
timeZone: timezone,
},
})
}
})
}

const cronList: Array<TaskMapper> = list() as Array<TaskMapper>
Expand Down
50 changes: 43 additions & 7 deletions src/main/resources/lib/ssb/cron/pushRss.ts
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
import { HttpRequestParams, HttpResponse } from '/lib/http-client'
import { getRssItemsStatkal } from '/lib/ssb/rss/statkal'
const { request } = __non_webpack_require__('/lib/http-client')
const { encryptRssNews } = __non_webpack_require__('/lib/cipher/cipherRss')
const { encryptRssNews, encryptRssStatkal } = __non_webpack_require__('/lib/cipher/cipherRss')

export function pushRssNews(): string {
const newsServiceUrl: string =
app.config && app.config['ssb.baseUrl']
? app.config['ssb.baseUrl'] + '/_/service/mimir/news'
: 'https:www.utv.ssb.no/_/service/mimir/news'
const rssNews: RssNews = getRssNews(newsServiceUrl)
: 'https://www.utv.ssb.no/_/service/mimir/news'
const rssNews: RssItems = getRssNews(newsServiceUrl)
if (rssNews.body !== null) {
const encryptedBody: string = encryptRssNews(rssNews.body)
return postRssNews(encryptedBody)
Expand All @@ -16,14 +17,24 @@ export function pushRssNews(): string {
}
}

function getRssNews(url: string): RssNews {
export function pushRssStatkal(): string {
const rssStatkal: string | null = getRssItemsStatkal()
if (rssStatkal !== null) {
const encryptedBody: string = encryptRssStatkal(rssStatkal)
return postRssStatkal(encryptedBody)
} else {
return 'Ingen publiseringer å pushe til rss/statkal'
}
}

function getRssNews(url: string): RssItems {
const requestParams: HttpRequestParams = {
url,
method: 'GET',
readTimeout: 40000,
}

const status: RssNews = {
const status: RssItems = {
body: null,
message: '',
}
Expand All @@ -50,7 +61,7 @@ function postRssNews(encryptedRss: string): string {
const rssNewsBaseUrl: string =
app.config && app.config['ssb.baseUrl']
? app.config['ssb.baseUrl'] + '/rss/populate/news'
: 'https:www.utv.ssb.no/rss/populate/news'
: 'https://www.utv.ssb.no/rss/populate/news'

const requestParams: HttpRequestParams = {
url: rssNewsBaseUrl,
Expand All @@ -70,11 +81,36 @@ function postRssNews(encryptedRss: string): string {
}
}

interface RssNews {
function postRssStatkal(encryptedRss: string): string {
const rssStatkalBaseUrl: string =
app.config && app.config['ssb.baseUrl']
? app.config['ssb.baseUrl'] + '/rss/populate/statkal'
: 'https://www.utv.ssb.no/rss/populate/statkal'

const requestParams: HttpRequestParams = {
url: rssStatkalBaseUrl,
method: 'POST',
body: encryptedRss,
}

try {
const pushRssStatkalResponse: HttpResponse = request(requestParams)
if (pushRssStatkalResponse.status === 200) {
return 'Push av RSS statkal OK'
} else {
return `Push av RSS statkal til ${rssStatkalBaseUrl} feilet - ${pushRssStatkalResponse.status} `
}
} catch (e) {
return 'Push av RSS statkal feilet - ' + e
}
}

interface RssItems {
body: string | null
message: string
}

export interface PushRSSLib {
pushRssNews: () => string
pushRssStatkal: () => string
}
1 change: 1 addition & 0 deletions src/main/resources/lib/ssb/repo/job.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ export enum JobNames {
STATISTICS_REFRESH_JOB = 'refresh statistics',
REFRESH_DATASET_JOB = 'Refresh dataset',
PUSH_RSS_NEWS = 'Push RSS news',
PUSH_RSS_STATKAL = 'Push RSS statkal',
REFRESH_DATASET_CALCULATOR_JOB = 'Refresh dataset calculators',
REFRESH_DATASET_NAMEGRAPH_JOB = 'Refresh dataset nameGraph',
REFRESH_DATASET_SDDS_TABLES_JOB = 'Refresh dataset for SDDS tables',
Expand Down
21 changes: 21 additions & 0 deletions src/main/resources/lib/ssb/repo/statisticVariant.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,22 @@ export function getStatisticVariantsFromRepo(
return res.hits.map((hit) => connectionStatisticRepo.get(hit.id) as ContentLight<Release>)
}

export function getUpcompingStatisticVariantsFromRepo(count?: number): ContentLight<Release>[] {
const connectionStatisticRepo: RepoConnection = getRepoConnectionStatistics()
const res = connectionStatisticRepo.query({
count: count ? count : 1000,
sort: 'publish.from DESC',
query: {
range: {
field: 'data.nextRelease',
gte: new Date().toISOString(),
},
},
})

return res.hits.map((hit) => connectionStatisticRepo.get(hit.id) as ContentLight<Release>)
}

export function fillRepo(statistics: Array<StatisticInListing>) {
if (getRepo(REPO_ID_STATREG_STATISTICS) === null) {
createRepo({
Expand Down Expand Up @@ -247,6 +263,7 @@ function prepareData({
allMainSubjectsStatistic,
allSubSubjectsStatistic,
}: CreateContentStatisticVariantParams): Release {
const statisticPath: string = statisticsContent?._path ? statisticsContent._path.split('/').slice(2).join('/') : ''
return {
statisticId: String(statistic.id),
variantId: String(variant.id),
Expand All @@ -270,7 +287,9 @@ function prepareData({
? capitalize(calculatePeriod(variant.frekvens, nextRelease.periodFrom, nextRelease.periodTo, language))
: '',
statisticContentId: statisticsContent?._id,
statisticPath: encodeURI(statisticPath),
articleType: 'statistics', // allows this content to be filtered together with `Article.articleType`,
contacts: statisticsContent?.data.contacts ? forceArray(statisticsContent?.data.contacts) : [],
mainSubjects: allMainSubjectsStatistic.map((subject) => subject.name).filter(notNullOrUndefined),
subSubjects: allSubSubjectsStatistic.map((subject) => subject.name).filter(notNullOrUndefined),
upcomingReleases: variant.upcomingReleases,
Expand Down Expand Up @@ -310,7 +329,9 @@ export interface Release {
nextRelease: string
nextPeriod: string
statisticContentId?: string
statisticPath: string
articleType: 'statistics'
contacts: string[]
mainSubjects: Array<string> | string | undefined
subSubjects: Array<string> | string | undefined
upcomingReleases?: Array<ReleasesInListing>
Expand Down
Loading

0 comments on commit f092de0

Please sign in to comment.