atproto self-host in docker #2026
Answered
by
bnewbold
syui
asked this question in
How to do it?
-
version
$ curl -sL ${pds}/xrpc/_health
{"version":"0.4.1"}$ git clone https://github.com/bluesky-social
$ cd bluesky-social
$ git reset --hard 53122b4b3e2e3e764e152e9a98d7ccd215900ef9docker compose
bsky=appview git clone https://github.com/bluesky-social/atproto
cd atproto
git clone https://github.com/did-method-plc/did-method-plc ./repos/did-method-plc
git clone https://github.com/bluesky-social/indigo ./repos/indigo
git clone https://github.com/bluesky-social/social-app ./repos/social-app
touch .plc.env .bsky.env .bgs.env .pds.env .db.env .mod.env .web.env .feed.envmkdir -p ./postgres/init
echo '-- plc
CREATE DATABASE plc;
GRANT ALL PRIVILEGES ON DATABASE plc TO postgres;
-- bgs
CREATE DATABASE bgs;
CREATE DATABASE carstore;
GRANT ALL PRIVILEGES ON DATABASE bgs TO postgres;
GRANT ALL PRIVILEGES ON DATABASE carstore TO postgres;
-- bsky(appview)
CREATE DATABASE bsky;
GRANT ALL PRIVILEGES ON DATABASE bsky TO postgres;
-- ozone(moderation)
CREATE DATABASE mod;
GRANT ALL PRIVILEGES ON DATABASE mod TO postgres;
-- pds
CREATE DATABASE pds;
GRANT ALL PRIVILEGES ON DATABASE pds TO postgres;
' >> ./postgres/init/init.sql
services:
plc:
ports:
- 2582:3000
build:
context: ./repos/did-method-plc/
dockerfile: packages/server/Dockerfile
env_file:
- ./.plc.env
depends_on:
- db
bgs:
ports:
- 2470:2470
build:
context: ./repos/indigo/
dockerfile: cmd/bigsky/Dockerfile
env_file:
- ./.bgs.env
volumes:
- ./data/bgs/:/data/
depends_on:
- db
bsky:
ports:
- 2584:3000
build:
context: ./
dockerfile: services/bsky/Dockerfile
env_file:
- ./.bsky.env
depends_on:
- db
- redis
bsky-daemon:
build:
context: ./
dockerfile: services/bsky/Dockerfile
command: node --enable-source-maps daemon.js
env_file:
- ./.bsky.env
depends_on:
- bsky
- db
- redis
bsky-indexer:
build:
context: ./
dockerfile: services/bsky/Dockerfile
command: node --enable-source-maps indexer.js
env_file:
- ./.bsky.env
volumes:
- ./data/bsky/cache/:/cache/
depends_on:
- bsky
- db
- redis
bsky-ingester:
build:
context: ./
dockerfile: services/bsky/Dockerfile
command: node --enable-source-maps ingester.js
env_file:
- ./.bsky.env
volumes:
- ./data/bsky/cache/:/cache/
depends_on:
- bsky
- db
- redis
mod:
ports:
- 2585:3000
build:
context: ./
dockerfile: services/ozone/Dockerfile
env_file:
- ./.mod.env
depends_on:
- db
mod-daemon:
build:
context: ./
dockerfile: services/ozone/Dockerfile
command: node --enable-source-maps daemon.js
env_file:
- ./.mod.env
depends_on:
- mod
- db
pds:
ports:
- 2583:3000
build:
context: ./
dockerfile: services/pds/Dockerfile
env_file:
- ./.pds.env
volumes:
- ./data/pds/:/data/
depends_on:
- db
social-app:
ports:
- 8100:8100
build:
context: ./repos/social-app/
dockerfile: Dockerfile
env_file:
- ./.web.env
command: "/usr/bin/bskyweb serve"
db:
image: postgres:latest
env_file:
- ./.db.env
volumes:
- ./postgres/init/:/docker-entrypoint-initdb.d/
- ./data/db/:/var/lib/postgresql/data/
redis:
image: redis
volumes:
- ./data/redis/:/data/other server$ git clone https://github.com/bluesky-social/feed-generator ./repos/feed-generator
feed-generator:
ports:
- 2586:3000
build:
context: ./repos/feed-generator/
restart: always
env_file:
- ./.feed.env
volumes:
- ./data/feed-generator/:/data/ search:
ports:
- 3999:3999
build:
context: ./repos/indigo/
dockerfile: cmd/palomar/Dockerfile
restart: always
env_file:
- ./.search.env
depends_on:
- databasehintservices:
plc:
depends_on:
db:
condition: service_startedservices:
plc:
depends_on:
db:
condition: service_healthy
db:
healthcheck:
# https://docs.docker.jp/compose/compose-file/compose-file-v3.html
test: ["CMD-SHELL", "pg_isready"]
interval: 10s
timeout: 5s
retries: 5services:
plc:
# https://docs.docker.jp/v19.03/config/container/start-containers-automatically.html
restart: alwaysservices:
db:
# https://hub.docker.com/_/postgres
image: postgres:16services:
db:
# postgresql://postgres:[email protected]:5432
ports:
- 5432:5432services:
db:
# postgresql://user:[email protected]/test
environment:
POSTGRES_USER: user
POSTGRES_DB: test
POSTGRES_PASSWORD: password $ docker compose up
$ docker compose downpostgresenv
# xxx
$ openssl ecparam --name secp256k1 --genkey --noout --outform DER | tail --bytes=+8 | head --bytes=32 | xxd --plain --cols 32appview : SERVICE_SIGNING_KEY/SERVER_DID
//SERVICE_SIGNING_KEY=xxx
//SERVER_DID=did:plc:xxx
static async create(cfg: BskyConfig): Promise<TestBsky> {
// packages/crypto/tests/keypairs.test.ts
const serviceKeypair = await Secp256k1Keypair.create({ exportable: true })
console.log(`ROTATION_KEY=${serviceKeypair.did()}`)
const exported = await serviceKeypair.export()
const plcClient = new PlcClient(cfg.plcUrl)
const port = cfg.port || (await getPort())
const url = `http://localhost:${port}`
const serverDid = await plcClient.createDid({
signingKey: serviceKeypair.did(),
rotationKeys: [serviceKeypair.did()],
handle: 'bsky.test',
pds: `http://localhost:${port}`,
signer: serviceKeypair,
})
console.log(`SERVER_DID=${serverDid}`)
const server = bsky.BskyAppView.create({
db,
redis: redisCache,
config,
algos: cfg.algos,
imgInvalidator: cfg.imgInvalidator,
signingKey: serviceKeypair,
})# https://web.plc.directory/api/redoc#operation/ResolveDid
url=https://plc.directory/did:plc:pyc2ihzpelxtg4cdkfzbhcv4
json='{ "type": "create", "signingKey": "did:key:zQ3shP5TBe1sQfSttXty15FAEHV1DZgcxRZNxvEWnPfLFwLxJ", "recoveryKey": "did:key:zQ3shhCGUqDKjStzuDxPkTxN6ujddP4RkEKJJouJGRRkaLGbg", "handle": "first-post.bsky.social", "service": "https://bsky.social", "prev": null, "sig": "yvN4nQYWTZTDl9nKSSyC5EC3nsF5g4S56OmRg9G6_-pM6FCItV2U2u14riiMGyHiCD86l6O-1xC5MPwf8vVsRw" }'
curl -X POST -H "Content-Type: application/json" -d "$json" $url | jq .pds : invitecodehost=example.com
admin_password="admin-pass"
url=https://$host/xrpc/com.atproto.server.createInviteCode
json="{\"useCount\":1}"
curl -X POST -u admin:${admin_password} -H "Content-Type: application/json" -d "$json" -sL $url | jq .bsky(appview) latest
$ curl -sL ${pds}/xrpc/_health
{"version":"0.4.24"}
bsky:
ports:
- 2584:3000
build:
context: ./repos/atproto/
dockerfile: services/bsky/Dockerfile
restart: always
env_file:
- ./envs/appview
volumes:
- ./data/appview/cache/:/cache/
command: node --enable-source-maps api.js
depends_on:
pds:
condition: service_healthy
db:
condition: service_healthy
redis:
condition: service_healthy
# bsky:
# ports:
# - 2584:3000
# build:
# context: ./repos/atproto/
# dockerfile: services/bsky/Dockerfile
# env_file:
# - ./envs/appview
# volumes:
# - ./data/appview/cache/:/cache/
# depends_on:
# db:
# condition: service_healthy
# redis:
# condition: service_healthy
#
# bsky-daemon:
# command: node --enable-source-maps daemon.js
# bsky-indexer:
# command: node --enable-source-maps indexer.js
# bsky-ingester:
# command: node --enable-source-maps ingester.js# caddy and ssl3
BSKY_REPO_PROVIDER=ws://pds:3000
BSKY_REPO_PROVIDER=ws://${host}
BSKY_BSYNC_HTTP_VERSION=1.1
BSKY_BSYNC_PORT=3002
BSKY_BSYNC_URL=http://bsky:3002
BSKY_COURIER_URL=http://fake-courier.example.invalid/
BSKY_DATAPLANE_HTTP_VERSION=1.1
BSKY_DATAPLANE_PORT=3001
BSKY_DATAPLANE_URLS=http://bsky:3001
BSKY_DB_POSTGRES_SCHEMA=bsky
BSKY_DB_POSTGRES_URL=postgres://postgres:postgres@db
BSKY_DID_PLC_URL=https://plc.${host}
BSKY_PUBLIC_URL=https://api.${host}
BSKY_SEARCH_URL=https://palomar.${host}
BSKY_SERVER_DID=did:web:api.${host}
BSKY_SERVICE_SIGNING_KEY=xxx
DEBUG_MODE=1
DID_PLC_URL=https://plc.${host}
ENABLE_MIGRATIONS=false
LOG_DESTINATION=1
LOG_ENABLED=true
MOD_SERVICE_DID=did:web:mod.${host}
NODE_ENV=development
REDIS_HOST=redis
#NODE_TLS_REJECT_UNAUTHORIZED=${NODE_TLS_REJECT_UNAUTHORIZED}
#GOINSECURE=${GOINSECURE}
#BSKY_LABELS_FROM_ISSUER_DIDS=${BSKY_LABELS_FROM_ISSUER_DIDS}
#PORT=2584
postgres://postgres:postgres@db/${BSKY_DB_POSTGRES_SCHEMA}
# compose.yml
bsky:
ports:
- 2584:3000special thanksthx ikuradon -> https://github.com/ikuradon/atproto-starter-kit |
Beta Was this translation helpful? Give feedback.
Answered by
bnewbold
Jan 8, 2024
Replies: 3 comments 4 replies
-
|
There is an out-of-date docker-compose file and README in the indigo repo that might be helpful: https://github.com/bluesky-social/indigo/tree/main/cmd/fakermaker Things are not in a great state for this right now. We plan to iterate on all the docs and dev support around self-hosting at the same time we open up federation, which means we aren't prioritizing updating docs on this until then. In particular, I think the pds service is probably pretty out of date compared to the sandbox PDS "distribution", and we need to get those synced back up, and that may involve breaking changes to the PDS config env vars. Also the version of the PDS that we (bluesky) runs is now behind a "gateway service", which is unnecessary complexity for most folks (but needs documentation). The relay (BGS) and PLC are pretty stable. The appview is in the middle of several large refactors, with big changes in feature branches which have not been merged yet. |
Beta Was this translation helpful? Give feedback.
0 replies
Answer selected by
syui
-
|
@syui did you succeed on self hosting with current codes? please update if you got progress. |
Beta Was this translation helpful? Give feedback.
4 replies
-
|
yes, my account https://web.syu.is/profile/syui.syu.is |
Beta Was this translation helpful? Give feedback.
-
|
could you share how you configure those? I also want to get self-hosting environment but failed configuring palomar (search service in indigo/cmd) |
Beta Was this translation helpful? Give feedback.
-
|
I have compiled information for self-hosting here. |
Beta Was this translation helpful? Give feedback.
-
|
hmm, finally I got my self-hosting environment for latest bluesky including bgs, plc, appview, pds, and others. |
Beta Was this translation helpful? Give feedback.
-
|
if bsky(appview) doesn't work |
Beta Was this translation helpful? Give feedback.
0 replies
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
There is an out-of-date docker-compose file and README in the indigo repo that might be helpful: https://github.com/bluesky-social/indigo/tree/main/cmd/fakermaker
Things are not in a great state for this right now. We plan to iterate on all the docs and dev support around self-hosting at the same time we open up federation, which means we aren't prioritizing updating docs on this until then.
In particular, I think the pds service is probably pretty out of date compared to the sandbox PDS "distribution", and we need to get those synced back up, and that may involve breaking changes to the PDS config env vars. Also the version of the PDS that we (bluesky) runs is now behind a "gateway servi…