MG-2456 - Refactor auth (absmach#2409)

- Set/Unset parent Group for Things and Channels (absmach#2486) - Move groups out of pkg (absmach#2493) - Separate Things authn and Channels authz (absmach#2496) Signed-off-by: Arvindh <[email protected]> NOISSUE - Add Publish/Subscribe to channels (absmach#2497) Signed-off-by: Arvindh <[email protected]> MG-2457 - Update auth tests (absmach#2503) Signed-off-by: Felix Gateru <[email protected]> MG-2477 - Replace Things with Clients (absmach#2508) Signed-off-by: Dusan Borovcanin <[email protected]> NOISSUE - Rename Things to Clients Signed-off-by: Felix Gateru <[email protected]> Signed-off-by: Arvindh <[email protected]>
arvindh123 · Nov 18, 2024 · 37e5c2c · 37e5c2c
1 parent d9f3bca
commit 37e5c2c
Show file tree

Hide file tree

Showing 508 changed files with 44,193 additions and 28,495 deletions.
diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml
@@ -19,7 +19,7 @@ on:
       - "journal/api/**"
       - "provision/api/**"
       - "readers/api/**"
-      - "things/api/**"
+      - "clients/api/**"
       - "users/api/**"
 
 env:
@@ -29,7 +29,7 @@ env:
   USER_SECRET: 12345678
   DOMAIN_NAME: demo-test
   USERS_URL: http://localhost:9002
-  THINGS_URL: http://localhost:9000
+  CLIENTS_URL: http://localhost:9000
   HTTP_ADAPTER_URL: http://localhost:8008
   INVITATIONS_URL: http://localhost:9020
   AUTH_URL: http://localhost:8189
@@ -65,8 +65,8 @@ jobs:
           export DOMAIN_ID=$(curl -sSX POST $DOMAINS_URL -H "Content-Type: application/json" -H "Authorization: Bearer $USER_TOKEN" -d "{\"name\":\"$DOMAIN_NAME\",\"alias\":\"$DOMAIN_NAME\"}" | jq -r .id)
           export USER_TOKEN=$(curl -sSX POST $TOKENS_URL -H "Content-Type: application/json" -d "{\"identity\": \"$USER_IDENTITY\",\"secret\": \"$USER_SECRET\",\"domain_id\": \"$DOMAIN_ID\"}" | jq -r .access_token)
           echo "USER_TOKEN=$USER_TOKEN" >> $GITHUB_ENV
-          export THING_SECRET=$(magistrala-cli provision test | /usr/bin/grep -Eo '"secret": "[^"]+"' | awk 'NR % 2 == 0' | sed 's/"secret": "\(.*\)"/\1/')
-          echo "THING_SECRET=$THING_SECRET" >> $GITHUB_ENV
+          export CLIENT_SECRET=$(magistrala-cli provision test | /usr/bin/grep -Eo '"secret": "[^"]+"' | awk 'NR % 2 == 0' | sed 's/"secret": "\(.*\)"/\1/')
+          echo "CLIENT_SECRET=$CLIENT_SECRET" >> $GITHUB_ENV
 
       - name: Check for changes in specific paths
         uses: dorny/paths-filter@v3
@@ -113,10 +113,10 @@ jobs:
               - "api/openapi/readers.yml"
               - "readers/api/**"
 
-            things:
+            clients:
               - ".github/workflows/api-tests.yml"
-              - "api/openapi/things.yml"
-              - "things/api/**"
+              - "api/openapi/clients.yml"
+              - "clients/api/**"
 
             users:
               - ".github/workflows/api-tests.yml"
@@ -133,12 +133,12 @@ jobs:
           report: false
           args: '--header "Authorization: Bearer ${{ env.USER_TOKEN }}" --contrib-openapi-formats-uuid --hypothesis-suppress-health-check=filter_too_much --stateful=links'
 
-      - name: Run Things API tests
-        if: steps.changes.outputs.things == 'true'
+      - name: Run Clients API tests
+        if: steps.changes.outputs.clients == 'true'
         uses: schemathesis/action@v1
         with:
-          schema: api/openapi/things.yml
-          base-url: ${{ env.THINGS_URL }}
+          schema: api/openapi/clients.yml
+          base-url: ${{ env.CLIENTS_URL }}
           checks: all
           report: false
           args: '--header "Authorization: Bearer ${{ env.USER_TOKEN }}" --contrib-openapi-formats-uuid --hypothesis-suppress-health-check=filter_too_much --stateful=links'
@@ -151,7 +151,7 @@ jobs:
           base-url: ${{ env.HTTP_ADAPTER_URL }}
           checks: all
           report: false
-          args: '--header "Authorization: Thing ${{ env.THING_SECRET }}" --contrib-openapi-formats-uuid --hypothesis-suppress-health-check=filter_too_much --stateful=links'
+          args: '--header "Authorization: Client ${{ env.CLIENT_SECRET }}" --contrib-openapi-formats-uuid --hypothesis-suppress-health-check=filter_too_much --stateful=links'
 
       - name: Run Invitations API tests
         if: steps.changes.outputs.invitations == 'true'

diff --git a/.github/workflows/check-generated-files.yml b/.github/workflows/check-generated-files.yml
@@ -49,8 +49,8 @@ jobs:
               - "users/clients.go"
               - "pkg/clients/clients.go"
               - "pkg/messaging/pubsub.go"
-              - "things/postgres/clients.go"
-              - "things/things.go"
+              - "clients/postgres/clients.go"
+              - "clients/clients.go"
               - "pkg/authz.go"
               - "pkg/authn.go"
               - "auth/domains.go"
@@ -132,9 +132,9 @@ jobs:
           mv ./users/mocks/repository.go ./users/mocks/repository.go.tmp
           mv ./users/mocks/service.go ./users/mocks/service.go.tmp
           mv ./pkg/messaging/mocks/pubsub.go ./pkg/messaging/mocks/pubsub.go.tmp
-          mv ./things/mocks/repository.go ./things/mocks/repository.go.tmp
-          mv ./things/mocks/service.go ./things/mocks/service.go.tmp
-          mv ./things/mocks/cache.go ./things/mocks/cache.go.tmp
+          mv ./clients/mocks/repository.go ./clients/mocks/repository.go.tmp
+          mv ./clients/mocks/service.go ./clients/mocks/service.go.tmp
+          mv ./clients/mocks/cache.go ./clients/mocks/cache.go.tmp
           mv ./auth/mocks/authz.go ./auth/mocks/authz.go.tmp
           mv ./auth/mocks/domains.go ./auth/mocks/domains.go.tmp
           mv ./auth/mocks/keys.go ./auth/mocks/keys.go.tmp
@@ -161,7 +161,7 @@ jobs:
           mv ./journal/mocks/repository.go ./journal/mocks/repository.go.tmp
           mv ./journal/mocks/service.go ./journal/mocks/service.go.tmp
           mv ./auth/mocks/domains_client.go ./auth/mocks/domains_client.go.tmp
-          mv ./things/mocks/things_client.go ./things/mocks/things_client.go.tmp
+          mv ./clients/mocks/clients_client.go ./clients/mocks/clients_client.go.tmp
           mv ./pkg/authz/mocks/authz.go ./pkg/authz/mocks/authz.go.tmp
           mv ./pkg/authn/mocks/authn.go ./pkg/authn/mocks/authn.go.tmp
 
@@ -183,9 +183,9 @@ jobs:
           check_mock_changes ./users/mocks/repository.go "Users Repository ./users/mocks/repository.go"
           check_mock_changes ./users/mocks/service.go "Users Service ./users/mocks/service.go"
           check_mock_changes ./pkg/messaging/mocks/pubsub.go "PubSub ./pkg/messaging/mocks/pubsub.go"
-          check_mock_changes ./things/mocks/repository.go "Things Repository ./things/mocks/repository.go"
-          check_mock_changes ./things/mocks/service.go "Things Service ./things/mocks/service.go"
-          check_mock_changes ./things/mocks/cache.go "Things Cache ./things/mocks/cache.go"
+          check_mock_changes ./clients/mocks/repository.go "Clients Repository ./clients/mocks/repository.go"
+          check_mock_changes ./clients/mocks/service.go "Clients Service ./clients/mocks/service.go"
+          check_mock_changes ./clients/mocks/cache.go "Clients Cache ./clients/mocks/cache.go"
           check_mock_changes ./auth/mocks/authz.go "Auth Authz ./auth/mocks/authz.go"
           check_mock_changes ./auth/mocks/domains.go "Auth Domains ./auth/mocks/domains.go"
           check_mock_changes ./auth/mocks/keys.go "Auth Keys ./auth/mocks/keys.go"
@@ -214,4 +214,4 @@ jobs:
           check_mock_changes ./journal/mocks/service.go "Journal Service ./journal/mocks/service.go"
           check_mock_changes ./auth/mocks/domains_client.go "Domains Service Client ./auth/mocks/domains_client.go"
           check_mock_changes ./auth/mocks/token_client.go "Token Service Client ./auth/mocks/token_client.go"
-          check_mock_changes ./things/mocks/things_client.go "Things Service Client things/mocks/things_client.go"
+          check_mock_changes .clients/mocks/clients_client.go "Clients Service Client clients/mocks/clients_client.go"
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -111,7 +111,7 @@ jobs:
               - "cmd/coap/**"
               - "auth.pb.go"
               - "auth_grpc.pb.go"
-              - "things/**"
+              - "clients/**"
               - "pkg/messaging/**"
 
             consumers:
@@ -140,7 +140,7 @@ jobs:
               - "cmd/http/**"
               - "auth.pb.go"
               - "auth_grpc.pb.go"
-              - "things/**"
+              - "clients/**"
               - "pkg/messaging/**"
               - "logger/**"
 
@@ -163,7 +163,7 @@ jobs:
               - "cmd/mqtt/**"
               - "auth.pb.go"
               - "auth_grpc.pb.go"
-              - "things/**"
+              - "clients/**"
               - "pkg/messaging/**"
               - "logger/**"
               - "pkg/events/**"
@@ -197,7 +197,7 @@ jobs:
               - "invitations/**"
               - "provision/**"
               - "readers/**"
-              - "things/**"
+              - "clients/**"
               - "users/**"
 
             pkg-transformers:
@@ -221,12 +221,12 @@ jobs:
               - "cmd/timescale-reader/**"
               - "auth.pb.go"
               - "auth_grpc.pb.go"
-              - "things/**"
+              - "clients/**"
               - "auth/**"
 
-            things:
-              - "things/**"
-              - "cmd/things/**"
+            clients:
+              - "clients/**"
+              - "cmd/clients/**"
               - "auth.pb.go"
               - "auth_grpc.pb.go"
               - "auth/**"
@@ -249,7 +249,7 @@ jobs:
               - "cmd/ws/**"
               - "auth.pb.go"
               - "auth_grpc.pb.go"
-              - "things/**"
+              - "clients/**"
               - "pkg/messaging/**"
 
       - name: Create coverage directory
@@ -366,10 +366,10 @@ jobs:
         run: |
           go test --race -v -count=1 -coverprofile=coverage/readers.out ./readers/...
 
-      - name: Run things tests
-        if: steps.changes.outputs.things == 'true' || steps.changes.outputs.workflow == 'true'
+      - name: Run clients tests
+        if: steps.changes.outputs.clients == 'true' || steps.changes.outputs.workflow == 'true'
         run: |
-          go test --race -v -count=1 -coverprofile=coverage/things.out ./things/...
+          go test --race -v -count=1 -coverprofile=coverage/clients.out ./clients/...
 
       - name: Run users tests
         if: steps.changes.outputs.users == 'true' || steps.changes.outputs.workflow == 'true'

diff --git a/Makefile b/Makefile
@@ -2,10 +2,10 @@
 # SPDX-License-Identifier: Apache-2.0
 
 MG_DOCKER_IMAGE_NAME_PREFIX ?= magistrala
-BUILD_DIR = build
-SERVICES = auth users things http coap ws postgres-writer postgres-reader timescale-writer \
+BUILD_DIR ?= build
+SERVICES = auth users clients groups channels domains http coap ws postgres-writer postgres-reader timescale-writer \
 	timescale-reader cli bootstrap mqtt provision certs invitations journal
-TEST_API_SERVICES = journal auth bootstrap certs http invitations notifiers provision readers things users
+TEST_API_SERVICES = journal auth bootstrap certs http invitations notifiers provision readers clients users
 TEST_API = $(addprefix test_api_,$(TEST_API_SERVICES))
 DOCKERS = $(addprefix docker_,$(SERVICES))
 DOCKERS_DEV = $(addprefix docker_dev_,$(SERVICES))
@@ -19,10 +19,14 @@ empty:=
 space:= $(empty) $(empty)
 # Docker compose project name should follow this guidelines: https://docs.docker.com/compose/reference/#use--p-to-specify-a-project-name
 DOCKER_PROJECT ?= $(shell echo $(subst $(space),,$(USER_REPO)) | tr -c -s '[:alnum:][=-=]' '_' | tr '[:upper:]' '[:lower:]')
-DOCKER_COMPOSE_COMMANDS_SUPPORTED := up down config
+DOCKER_COMPOSE_COMMANDS_SUPPORTED := up down config restart
 DEFAULT_DOCKER_COMPOSE_COMMAND  := up
 GRPC_MTLS_CERT_FILES_EXISTS = 0
 MOCKERY_VERSION=v2.43.2
+INTERNAL_PROTO_GEN_OUT_DIR=internal/grpc
+INTERNAL_PROTO_DIR=internal/proto
+INTERNAL_PROTO_FILES := $(shell find $(INTERNAL_PROTO_DIR) -name "*.proto" | sed 's|$(INTERNAL_PROTO_DIR)/||')
+
 ifneq ($(MG_MESSAGE_BROKER_TYPE),)
     MG_MESSAGE_BROKER_TYPE := $(MG_MESSAGE_BROKER_TYPE)
 else
@@ -138,8 +142,8 @@ define test_api_service
 		exit 1; \
 	fi
 
-	@if [ "$(svc)" = "http" ] && [ -z "$(THING_SECRET)" ]; then \
-		echo "THING_SECRET is not set"; \
+	@if [ "$(svc)" = "http" ] && [ -z "$(CLIENT_SECRET)" ]; then \
+		echo "CLIENT_SECRET is not set"; \
 		echo "Please set it to a valid secret"; \
 		exit 1; \
 	fi
@@ -148,7 +152,7 @@ define test_api_service
 		st run api/openapi/$(svc).yml \
 		--checks all \
 		--base-url $(2) \
-		--header "Authorization: Thing $(THING_SECRET)" \
+		--header "Authorization: Client $(CLIENT_SECRET)" \
 		--contrib-openapi-formats-uuid \
 		--hypothesis-suppress-health-check=filter_too_much \
 		--stateful=links; \
@@ -164,7 +168,7 @@ define test_api_service
 endef
 
 test_api_users: TEST_API_URL := http://localhost:9002
-test_api_things: TEST_API_URL := http://localhost:9000
+test_api_clients: TEST_API_URL := http://localhost:9000
 test_api_http: TEST_API_URL := http://localhost:8008
 test_api_invitations: TEST_API_URL := http://localhost:9020
 test_api_auth: TEST_API_URL := http://localhost:8189
@@ -179,7 +183,8 @@ $(TEST_API):
 
 proto:
 	protoc -I. --go_out=. --go_opt=paths=source_relative pkg/messaging/*.proto
-	protoc -I. --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative ./*.proto
+	mkdir -p $(INTERNAL_PROTO_GEN_OUT_DIR)
+	protoc -I $(INTERNAL_PROTO_DIR) --go_out=$(INTERNAL_PROTO_GEN_OUT_DIR) --go_opt=paths=source_relative --go-grpc_out=$(INTERNAL_PROTO_GEN_OUT_DIR) --go-grpc_opt=paths=source_relative $(INTERNAL_PROTO_FILES)
 
 $(FILTERED_SERVICES):
 	$(call compile_service,$(@))
@@ -218,7 +223,7 @@ rundev:
 	cd scripts && ./run.sh
 
 grpc_mtls_certs:
-	$(MAKE) -C docker/ssl auth_grpc_certs things_grpc_certs
+	$(MAKE) -C docker/ssl auth_grpc_certs clients_grpc_certs
 
 check_tls:
 ifeq ($(GRPC_TLS),true)
@@ -244,7 +249,7 @@ check_certs: check_mtls check_tls
 ifeq ($(GRPC_MTLS_CERT_FILES_EXISTS),0)
 ifeq ($(filter true,$(GRPC_MTLS) $(GRPC_TLS)),true)
 ifeq ($(filter $(DEFAULT_DOCKER_COMPOSE_COMMAND),$(DOCKER_COMPOSE_COMMAND)),$(DEFAULT_DOCKER_COMPOSE_COMMAND))
-	$(MAKE) -C docker/ssl auth_grpc_certs things_grpc_certs
+	$(MAKE) -C docker/ssl auth_grpc_certs clients_grpc_certs
 endif
 endif
 endif
@@ -257,3 +262,6 @@ run_addons: check_certs
 	@for SVC in $(RUN_ADDON_ARGS); do \
 		MG_ADDONS_CERTS_PATH_PREFIX="../."  docker compose -f docker/addons/$$SVC/docker-compose.yml -p $(DOCKER_PROJECT) --env-file ./docker/.env $(DOCKER_COMPOSE_COMMAND) $(args) & \
 	done
+
+run_live: check_certs
+	GOPATH=$(go env GOPATH) docker compose  -f docker/docker-compose.yml -f docker/docker-compose-live.yaml   --env-file docker/.env -p $(DOCKER_PROJECT) $(DOCKER_COMPOSE_COMMAND) $(args)
diff --git a/README.md b/README.md
@@ -12,7 +12,7 @@
 
 Magistrala is modern, scalable, secure, open-source, and patent-free IoT cloud platform written in Go.
 
-It accepts user and thing (sensor, actuator, application) connections over various network protocols (i.e. HTTP, MQTT, WebSocket, CoAP), thus making a seamless bridge between them. It is used as the IoT middleware for building complex IoT solutions.
+It accepts user and client (sensor, actuator, application) connections over various network protocols (i.e. HTTP, MQTT, WebSocket, CoAP), thus making a seamless bridge between them. It is used as the IoT middleware for building complex IoT solutions.
 
 For more details, check out the [official documentation][docs].
 For extra bits and services see [our contrib repository][contrib].

diff --git a/api/asyncapi/mqtt.yml b/api/asyncapi/mqtt.yml
@@ -17,8 +17,8 @@ info:
   license:
     name: Apache 2.0
     url: 'https://github.com/absmach/magistrala/blob/main/LICENSE'
-  
-  
+
+
 defaultContentType: application/json
 
 servers:
@@ -33,7 +33,7 @@ servers:
         enum:
           - '1883'
           - '8883'
-    security: 
+    security:
       - user-password: []
 
 channels:
@@ -45,7 +45,7 @@ channels:
         required: true
       subtopic:
         $ref: '#/components/parameters/subtopic'
-        in: path 
+        in: path
         required: false
 
     publish:
@@ -88,7 +88,7 @@ components:
 
   parameters:
     channelID:
-      description: Channel ID connected to the Thing ID defined in the username.
+      description: Channel ID connected to the Client ID defined in the username.
       schema:
         type: string
         format: uuid
@@ -97,13 +97,13 @@ components:
       schema:
         type: string
         default: ''
-  
+
   securitySchemes:
     user-password:
       type: userPassword
       description: |
-        username is thing ID connected to the channel defined in the mqtt topic and 
-        password is thing key corresponding to the thing ID 
+        username is client ID connected to the channel defined in the mqtt topic and
+        password is client secret corresponding to the client ID
 
   operationTraits:
     mqtt:

diff --git a/api/asyncapi/websocket.yml b/api/asyncapi/websocket.yml
@@ -126,7 +126,7 @@ components:
         ```
   parameters:
     channelID:
-      description: Channel ID connected to the Thing ID defined in the username.
+      description: Channel ID connected to the Client ID defined in the username.
       schema:
         type: string
         format: uuid
@@ -141,4 +141,4 @@ components:
       scheme: bearer
       bearerFormat: uuid
       description: |
-        * Thing access: "Authorization: Thing <thing_key>"
+        * Client access: "Authorization: Client <client_key>"
diff --git a/api/openapi/auth.yml b/api/openapi/auth.yml
@@ -596,7 +596,7 @@ components:
         metadata:
           type: object
           example: { "domain": "example.com" }
-          description: Arbitrary, object-encoded thing's data.
+          description: Arbitrary, object-encoded client's data.
         alias:
           type: string
           example: domain alias
@@ -684,7 +684,7 @@ components:
           type: string
           description: |
             Specifies an object field for the field.
-            Object indicates application objects such as ThingID.
+            Object indicates application objects such as Client ID.
         subjects:
           type: array
           minItems: 1