Skip to content

Commit

Permalink
feat(jaen-pages): switch from ipfs to osg
Browse files Browse the repository at this point in the history
Due to availability issues, all functionalities related to IPFS have been
rewritten to use OSG (open-storage-gateway: http://osg.snek.at).
  • Loading branch information
schettn authored Nov 22, 2021
1 parent 8c7d908 commit 3e9df13
Show file tree
Hide file tree
Showing 9 changed files with 113 additions and 149 deletions.
129 changes: 9 additions & 120 deletions examples/my-gatsby-site/jaen-pages.json
Original file line number Diff line number Diff line change
@@ -1,139 +1,28 @@
{
"pages": {
"SitePage /30896cb4-b20a-48c5-bcde-740920566241": {
"context": {
"createdAt": "2021-09-14T10:14:21.623Z",
"fileUrl": "https://cloudflare-ipfs.com/ipfs/QmQDMy8MF2esgevUreoLRWEtGVXFMAz5ys1vK8BZtvG24X"
},
"migrations": [
{
"fileUrl": "https://cloudflare-ipfs.com/ipfs/QmR3zkS2HnGiL1BNfeRmAi9CVQKepo9ZKQSXd5kNuDaFjb",
"createdAt": "2021-09-11T23:12:46.731Z"
},
{
"fileUrl": "https://cloudflare-ipfs.com/ipfs/QmPbyHY9aSP9xmryMnMF2M3RrvFVZ8t8ai5kNbprNfPyoc",
"createdAt": "2021-09-14T04:17:52.257Z"
},
{
"fileUrl": "https://cloudflare-ipfs.com/ipfs/QmTiZPiXxHWrLTxVYehE3dv7RiKmL1E8JYffQygxaKgsWC",
"createdAt": "2021-09-14T09:43:48.175Z"
},
{
"fileUrl": "https://cloudflare-ipfs.com/ipfs/Qmdh4qmwFSefUHVPV7QSQzJsktcFrFAnPRPoXNzqshhXTE",
"createdAt": "2021-09-14T10:14:21.623Z"
}
]
},
"SitePage /": {
"context": {
"fileUrl": "https://cloudflare-ipfs.com/ipfs/QmWVYPWK3UK3CtXtEs2pjiteh9xjnn2PgXdBY7zYf9hqgD",
"createdAt": "2021-09-11T23:12:46.731Z"
},
"migrations": [
{
"fileUrl": "https://cloudflare-ipfs.com/ipfs/QmWVYPWK3UK3CtXtEs2pjiteh9xjnn2PgXdBY7zYf9hqgD",
"createdAt": "2021-09-11T23:12:46.731Z"
}
]
},
"SitePage /091bbaf6-aee5-4d85-b564-aab57627658c": {
"context": {
"fileUrl": "https://cloudflare-ipfs.com/ipfs/QmQC79LZYGraNHUrGzfNcHPvehR7HfzLXrTKTW7HP72Dfw",
"createdAt": "2021-09-11T23:12:46.731Z"
},
"migrations": [
{
"fileUrl": "https://cloudflare-ipfs.com/ipfs/QmQC79LZYGraNHUrGzfNcHPvehR7HfzLXrTKTW7HP72Dfw",
"createdAt": "2021-09-11T23:12:46.731Z"
}
]
},
"SitePage /3ab722c8-db8f-41b9-8277-badbf0ad3261": {
"context": {
"createdAt": "2021-09-16T03:36:18.858Z",
"fileUrl": "https://cloudflare-ipfs.com/ipfs/QmYjn9Kgkv85YpMgrzMfjAoTDXaNkdu7dhgDHGTiXSvkyF"
},
"migrations": [
{
"fileUrl": "https://cloudflare-ipfs.com/ipfs/Qmb9MsnbuMH3Kynn6191p7zPa7BZ26sbwTVjuJvvNhkmme",
"createdAt": "2021-09-11T23:12:46.731Z"
},
{
"fileUrl": "https://cloudflare-ipfs.com/ipfs/QmQe7YuL8M2QMcfPbRcDhDuGXVyGgUGU5fmnFQ49jzXsgN",
"createdAt": "2021-09-16T03:36:18.858Z"
}
]
},
"SitePage /876b413b-7f3d-408e-9674-ecf57f65dad3": {
"context": {
"fileUrl": "https://cloudflare-ipfs.com/ipfs/QmXCwJtbYi1WSwZM8aGzyaEuLpwZ38Fv4WJwW4ii94B6re",
"createdAt": "2021-09-16T03:36:18.858Z"
},
"migrations": [
{
"fileUrl": "https://cloudflare-ipfs.com/ipfs/QmXCwJtbYi1WSwZM8aGzyaEuLpwZ38Fv4WJwW4ii94B6re",
"createdAt": "2021-09-16T03:36:18.858Z"
}
]
},
"SitePage /580d1d1c-61c1-4454-aa38-acb4ce83ac4d": {
"context": {
"fileUrl": "https://cloudflare-ipfs.com/ipfs/Qma4i8rrAc9wbEJc5GGqeuEWhxVU67WAi9ykv6tQev4X4C",
"createdAt": "2021-09-16T03:36:18.858Z"
},
"migrations": [
{
"fileUrl": "https://cloudflare-ipfs.com/ipfs/Qma4i8rrAc9wbEJc5GGqeuEWhxVU67WAi9ykv6tQev4X4C",
"createdAt": "2021-09-16T03:36:18.858Z"
}
]
},
"SitePage /cbf6930d-cb91-4dd9-a306-0ca075b7e0df": {
"context": {
"fileUrl": "https://cloudflare-ipfs.com/ipfs/QmUSCni8VUmuVbC6FnXnLmoyZNmmth6CHyefTMbGM1dYWr",
"createdAt": "2021-09-16T03:45:02.787Z"
},
"migrations": [
{
"fileUrl": "https://cloudflare-ipfs.com/ipfs/QmUSCni8VUmuVbC6FnXnLmoyZNmmth6CHyefTMbGM1dYWr",
"createdAt": "2021-09-16T03:45:02.787Z"
}
]
},
"SitePage /testpage/": {
"context": {
"fileUrl": "https://cloudflare-ipfs.com/ipfs/QmQ3Rbs4rq3wXqNT9wwR2QbQu7EgDn1KaHtTSQuZV9j3Eh",
"createdAt": "2021-09-16T03:45:02.787Z"
"fileUrl": "https://osg.snek.at/storage/BQACAgQAAxkDAANoYZqXG2kWlz1a5tKwxcso81HBKGAAAmELAAL6WdhQDo3zaRx4J8ciBA",
"createdAt": "2021-11-21T18:59:39.264Z"
},
"migrations": [
{
"fileUrl": "https://cloudflare-ipfs.com/ipfs/QmQ3Rbs4rq3wXqNT9wwR2QbQu7EgDn1KaHtTSQuZV9j3Eh",
"createdAt": "2021-09-16T03:45:02.787Z"
"fileUrl": "https://osg.snek.at/storage/BQACAgQAAxkDAANoYZqXG2kWlz1a5tKwxcso81HBKGAAAmELAAL6WdhQDo3zaRx4J8ciBA",
"createdAt": "2021-11-21T18:59:39.264Z"
}
]
}
},
"site": {
"context": {
"createdAt": "2021-09-14T21:06:26.349Z",
"fileUrl": "https://cloudflare-ipfs.com/ipfs/QmXNmfbdvURoSrGFCffEDTcPbB2Q4tHxUHA11mMX11xtNq"
},
"migrations": [
{
"createdAt": "2021-09-14T21:06:26.349Z",
"fileUrl": "https://cloudflare-ipfs.com/ipfs/QmXNmfbdvURoSrGFCffEDTcPbB2Q4tHxUHA11mMX11xtNq"
}
]
},
"site": {},
"snekFinder": {
"context": {
"createdAt": "2021-09-27T00:17:50.286Z",
"fileUrl": "https://cloudflare-ipfs.com/ipfs/QmQhRzSuengWU9m1qaQTzKY4STK3TxZoaJTw7GWpPArmdP"
"createdAt": "2021-11-21T18:59:39.264Z",
"fileUrl": "https://osg.snek.at/storage/BQACAgQAAxkDAANlYZqV28a7097qmMjvd00iFx7M5tgAAl4LAAL6WdhQsW753yRCkaQiBA"
},
"migrations": [
{
"createdAt": "2021-09-27T00:17:50.286Z",
"fileUrl": "https://cloudflare-ipfs.com/ipfs/QmQhRzSuengWU9m1qaQTzKY4STK3TxZoaJTw7GWpPArmdP"
"createdAt": "2021-11-21T18:59:39.264Z",
"fileUrl": "https://osg.snek.at/storage/BQACAgQAAxkDAANlYZqV28a7097qmMjvd00iFx7M5tgAAl4LAAL6WdhQsW753yRCkaQiBA"
}
]
}
Expand Down
6 changes: 3 additions & 3 deletions packages/jaen-pages/jaen-register.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ export default {
},
registerCallbacks: {
onPublish: async () => {
const {upload} = await import('./src/ipfs')
const {upload} = await import('./src/storage')

const {allSitePage} = storageGet()

Expand All @@ -89,14 +89,14 @@ export default {
if (nodes) {
for (const [id, node] of Object.entries(nodes)) {
const path = resolvePath(id, allNodes as any)
const paylaod = JSON.stringify({...node, path})
const paylaod = {...node, path}

const url = await upload(paylaod)
newPages[id] = {context: {fileUrl: url, createdAt}}
}
}

const siteMetadataPayload = JSON.stringify(state.site.siteMetadata)
const siteMetadataPayload = state.site.siteMetadata

const newSiteMetadata = siteMetadataPayload
? await upload(siteMetadataPayload)
Expand Down
2 changes: 1 addition & 1 deletion packages/jaen-pages/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
"@reduxjs/toolkit": "^1.6.1",
"@snek-at/jaen": "^2.0.0",
"@snek-at/jaen-shared-ui": "2.0.0",
"@snek-at/snek-finder": "^1.0.0-beta.16",
"@snek-at/snek-finder": "^1.0.0-beta.17",
"deepmerge": "^4.2.2",
"electron": "^14.0.0",
"esm": "^3.2.25",
Expand Down
6 changes: 3 additions & 3 deletions packages/jaen-pages/src/cli/publish.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ export const mergeBaseWithMigration = async (
baseEntity: JaenPagesEntityWithMigrations | undefined,
migrationEntity: JaenPagesEntity
) => {
const {upload} = await import('../ipfs') // do not change the import path
const {upload} = await import('../storage') // do not change the import path
const migrationContext = migrationEntity.context

// check if baseEntity is not a empty object
Expand All @@ -31,8 +31,8 @@ export const mergeBaseWithMigration = async (

// console.log('fetch done', typeof baseData, typeof migrationData)

const mergedData = merge(baseData, migrationData)
const fileUrl = await upload(JSON.stringify(mergedData))
const mergedData = merge(baseData, migrationData) as object
const fileUrl = await upload(mergedData)

const context = {
createdAt: migrationContext.createdAt,
Expand Down
29 changes: 23 additions & 6 deletions packages/jaen-pages/src/containers/SnekFinder/index.tsx
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import SnekFinder from '@snek-at/snek-finder'
import IPFSBackend from '@snek-at/snek-finder/lib/backends/IPFSBackend'
import OSGBackend from '@snek-at/snek-finder/lib/backends/OSGBackend'
import * as actions from '@store/actions/sfActions'
import {store, useAppDispatch} from '@store/index'
import {withRedux} from '@store/withRedux'
Expand All @@ -24,21 +24,38 @@ const Finder: React.FC<FinderProps> = ({mode = 'browser', ...props}) => {
}
`)

const [loading, setLoading] = React.useState(true)

const initBackendLink =
data.jaenPagesInitials.snekFinder.initBackendLink ||
store.getState().sf.initBackendLink
store.getState().sf.initBackendLink ||
data.jaenPagesInitials.snekFinder.initBackendLink

IPFSBackend.onBackendLinkChange = (link: string) => {
OSGBackend.onBackendLinkChange = (link: string) => {
dispatch(actions.setBackendLink(link))
}

React.useEffect(() => {
if (initBackendLink) {
IPFSBackend.initBackendLink = initBackendLink
OSGBackend.initBackendLink = initBackendLink

const run = async () => {
const response = await (await fetch(initBackendLink)).json()

localStorage.setItem(
'snek-finder-osg-backend-root',
JSON.stringify(response)
)

setLoading(false)
}

run()
}
})

return <SnekFinder backend={IPFSBackend} mode={mode} {...props} />
return loading ? null : (
<SnekFinder backend={OSGBackend} mode={mode} {...props} />
)
}

export default withRedux(Finder)
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,9 @@ const Button: React.FC = () => {
const isDisabled = core.getAuthState().isGuest

const publish = async () => {
const {upload} = await import('@src/ipfs')
const {upload} = await import('@src/storage')
const res = (await core.onPublish()) as {'jaen-pages': object}

const url = await upload(JSON.stringify(res['jaen-pages']))
const url = await upload(res['jaen-pages'])

const publishRes = await Bridge.session.mutate<{
jaenPublishFormPage: {result: string}
Expand Down
12 changes: 0 additions & 12 deletions packages/jaen-pages/src/ipfs.ts

This file was deleted.

44 changes: 44 additions & 0 deletions packages/jaen-pages/src/storage.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
export const upload = async (
payload: object | Blob | File,
backend: 'IPFS' | 'OSG' = 'OSG'
) => {
if (backend === 'IPFS') {
const ipfsClient = await import('ipfs-http-client')

const ipfs = ipfsClient.create({
host: 'ipfs.infura.io',
port: 5001,
protocol: 'https'
})

const {cid} = await ipfs.add(
typeof payload === 'object' ? JSON.stringify(payload) : payload
)
return `https://cloudflare-ipfs.com/ipfs/${cid.toString()}`
} else {
const url = 'https://osg.snek.at/storage'

const formData = new FormData()

// payload to blob
if (payload instanceof Blob || payload instanceof File) {
formData.append('file', payload)
} else {
formData.append(
'file',
new File([JSON.stringify(payload)], 'jaen-index.json', {
type: 'application/json'
})
)
}

const resp = await fetch(url, {
body: formData,
method: 'POST'
})

const json = await resp.json()

return `${url}/${json.file_id}`
}
}
29 changes: 28 additions & 1 deletion yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -4686,7 +4686,7 @@
dependencies:
lottie-web "^5.7.12"

"@snek-at/[email protected]", "@snek-at/snek-finder@^1.0.0-beta.16":
"@snek-at/[email protected]":
version "1.0.0-beta.16"
resolved "https://registry.yarnpkg.com/@snek-at/snek-finder/-/snek-finder-1.0.0-beta.16.tgz#aea560a4b530633a7e34ca94cc4a86e8622f8bed"
integrity sha512-BVUFcxMmmZ2HAm+gs5EFhv0AaO7Bix8x8uhcchh8PNnxX/EnilCn6EBDiygj7KVDRqZMec2I5nqL9pcP0Gvo+Q==
Expand All @@ -4713,6 +4713,33 @@
react-table "^7.7.0"
react-zoom-pan-pinch "^2.1.3"

"@snek-at/snek-finder@^1.0.0-beta.17":
version "1.0.0-beta.17"
resolved "https://registry.yarnpkg.com/@snek-at/snek-finder/-/snek-finder-1.0.0-beta.17.tgz#b60d0d9bfd044fbe8127df7bd030bc4e830c5805"
integrity sha512-DKcdJGFmg+DEPOxb7UApxkpH1e3xouOu5djd/2CPnlFjb+4Je3FulMSpGIF4tM4mLDUPHVl39p9ITBMrXW6Yfg==
dependencies:
"@chakra-ui/icons" "^1.0.14"
"@chakra-ui/react" "^1.6.6"
"@emotion/react" "^11"
"@emotion/styled" "^11"
"@react-icons/all-files" "^4.1.0"
"@react-pdf-viewer/core" "^2.7.1"
"@react-pdf-viewer/default-layout" "^2.7.1"
"@react-pdf-viewer/print" "^2.7.1"
"@toast-ui/react-image-editor" "^3.15.0"
filerobot-image-editor "^3.12.17"
framer-motion "^4"
immutability-helper "^3.1.1"
ipfs-http-client "^52.0.0"
pdf2pic "^2.1.4"
react "^17.0.2"
react-dnd "^14.0.2"
react-dnd-html5-backend "^14.0.0"
react-dropzone "^11.3.4"
react-pdf "^5.3.2"
react-table "^7.7.0"
react-zoom-pan-pinch "^2.1.3"

"@snek-at/storybook-addon-chakra-ui@^1.0.0-beta.2":
version "1.0.0-beta.2"
resolved "https://registry.yarnpkg.com/@snek-at/storybook-addon-chakra-ui/-/storybook-addon-chakra-ui-1.0.0-beta.2.tgz#4d73b00cf051a5161c842d37560b8f976fd15dd8"
Expand Down

0 comments on commit 3e9df13

Please sign in to comment.