chore: add artefact metadata to the module

block · Dec 6, 2024 · f8bfedb · f8bfedb
1 parent 246e0ea
commit f8bfedb
Show file tree

Hide file tree

Showing 14 changed files with 1,090 additions and 818 deletions.
diff --git a/backend/protos/xyz/block/ftl/schema/v1/schema.pb.go b/backend/protos/xyz/block/ftl/schema/v1/schema.pb.go
diff --git a/backend/protos/xyz/block/ftl/schema/v1/schema.proto b/backend/protos/xyz/block/ftl/schema/v1/schema.proto
@@ -176,6 +176,7 @@ message Map {
 message Metadata {
   oneof value {
     MetadataAlias alias = 5;
+    MetadataArtefact artefact = 14;
     MetadataCalls calls = 1;
     MetadataConfig config = 10;
     MetadataCronJob cron_job = 3;
@@ -197,6 +198,13 @@ message MetadataAlias {
   string alias = 3;
 }
 
+message MetadataArtefact {
+  optional Position pos = 1;
+  string path = 2;
+  string digest = 3;
+  bool executable = 4;
+}
+
 message MetadataCalls {
   optional Position pos = 1;
   repeated Ref calls = 2;
@@ -272,6 +280,7 @@ message Module {
   bool builtin = 3;
   string name = 4;
   repeated Decl decls = 5;
+  repeated Metadata metadata = 6;
   ModuleRuntime runtime = 31634;
 }
 

diff --git a/frontend/console/src/protos/xyz/block/ftl/schema/v1/schema_pb.ts b/frontend/console/src/protos/xyz/block/ftl/schema/v1/schema_pb.ts
diff --git a/internal/schema/data.go b/internal/schema/data.go
@@ -127,7 +127,7 @@ func (d *Data) Monomorphise(ref *Ref) (*Data, error) {
 			*EnumVariant, Value, *IntValue, *StringValue, *TypeValue, Symbol,
 			Named, *TypeAlias, *Topic, *MetadataSubscriber, *MetadataTypeMap,
 			*MetadataEncoding, *MetadataPublisher, *MetadataSQLMigration, *DSNDatabaseConnector, *DatabaseRuntime,
-			DatabaseConnector, *AWSIAMAuthDatabaseConnector, *DatabaseRuntimeConnections:
+			DatabaseConnector, *AWSIAMAuthDatabaseConnector, *DatabaseRuntimeConnections, *MetadataArtefact:
 		}
 		return next()
 	})

diff --git a/internal/schema/jsonschema.go b/internal/schema/jsonschema.go
@@ -195,7 +195,7 @@ func nodeToJSSchema(node Node, refs map[RefKey]*Ref) *jsonschema.Schema {
 		*StringValue, *IntValue, *TypeValue, *Config, *Secret, Symbol, Named,
 		*MetadataRetry, *Topic, *MetadataSubscriber, *MetadataTypeMap,
 		*MetadataEncoding, *MetadataPublisher, *MetadataSQLMigration, *DSNDatabaseConnector, *DatabaseRuntime,
-		*AWSIAMAuthDatabaseConnector, DatabaseConnector, *DatabaseRuntimeConnections:
+		*AWSIAMAuthDatabaseConnector, DatabaseConnector, *DatabaseRuntimeConnections, *MetadataArtefact:
 		panic(fmt.Sprintf("unsupported node type %T", node))
 
 	default:

diff --git a/internal/schema/metadataartefact.go b/internal/schema/metadataartefact.go
@@ -0,0 +1,38 @@
+package schema
+
+import (
+	"fmt"
+
+	"google.golang.org/protobuf/reflect/protoreflect"
+
+	schemapb "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/schema/v1"
+)
+
+//protobuf:14
+type MetadataArtefact struct {
+	Pos Position `parser:"" protobuf:"1,optional"`
+
+	Path       string `parser:"'+' 'artefact' Whitespace @String" protobuf:"2"`
+	Digest     string `parser:"@String" protobuf:"3"`
+	Executable bool   `parser:"@'executable'?" protobuf:"4"`
+}
+
+var _ Metadata = (*MetadataArtefact)(nil)
+
+func (m *MetadataArtefact) Position() Position { return m.Pos }
+
+func (m *MetadataArtefact) String() string {
+	return fmt.Sprintf("+artefact %q %q %t", m.Path, m.Digest, m.Executable)
+}
+
+func (m *MetadataArtefact) ToProto() protoreflect.ProtoMessage {
+	return &schemapb.MetadataArtefact{
+		Pos:        posToProto(m.Pos),
+		Executable: m.Executable,
+		Path:       m.Path,
+		Digest:     m.Digest,
+	}
+}
+
+func (m *MetadataArtefact) schemaChildren() []Node { return nil }
+func (m *MetadataArtefact) schemaMetadata()        {}
diff --git a/internal/schema/module.go b/internal/schema/module.go
@@ -21,10 +21,11 @@ import (
 type Module struct {
 	Pos Position `parser:"" protobuf:"1,optional"`
 
-	Comments []string `parser:"@Comment*" protobuf:"2"`
-	Builtin  bool     `parser:"@'builtin'?" protobuf:"3"`
-	Name     string   `parser:"'module' @Ident '{'" protobuf:"4"`
-	Decls    []Decl   `parser:"@@* '}'" protobuf:"5"`
+	Comments []string   `parser:"@Comment*" protobuf:"2"`
+	Builtin  bool       `parser:"@'builtin'?" protobuf:"3"`
+	Name     string     `parser:"'module' @Ident '{'" protobuf:"4"`
+	Metadata []Metadata `parser:"@@*" protobuf:"6"`
+	Decls    []Decl     `parser:"@@* '}'" protobuf:"5"`
 
 	Runtime *ModuleRuntime `protobuf:"31634" parser:""`
 }
@@ -235,6 +236,7 @@ func (m *Module) ToProto() proto.Message {
 		Comments: m.Comments,
 		Decls:    declListToProto(m.Decls),
 		Runtime:  runtime,
+		Metadata: metadataListToProto(m.Metadata),
 	}
 }
 
@@ -259,6 +261,7 @@ func ModuleFromProto(s *schemapb.Module) (*Module, error) {
 		Comments: s.Comments,
 		Decls:    declListToSchema(s.Decls),
 		Runtime:  ModuleRuntimeFromProto(s.Runtime),
+		Metadata: metadataListToSchema(s.Metadata),
 	}
 	return module, ValidateModule(module)
 }

diff --git a/internal/schema/parser.go b/internal/schema/parser.go
@@ -23,7 +23,7 @@ var (
 	}
 	metadataUnion = []Metadata{&MetadataCalls{}, &MetadataConfig{}, &MetadataIngress{}, &MetadataCronJob{},
 		&MetadataDatabases{}, &MetadataAlias{}, &MetadataRetry{}, &MetadataSecrets{}, &MetadataSubscriber{},
-		&MetadataTypeMap{}, &MetadataEncoding{}, &MetadataPublisher{}, &MetadataSQLMigration{}}
+		&MetadataTypeMap{}, &MetadataEncoding{}, &MetadataPublisher{}, &MetadataSQLMigration{}, &MetadataArtefact{}}
 	ingressUnion = []IngressPathComponent{&IngressPathLiteral{}, &IngressPathParameter{}}
 	valueUnion   = []Value{&StringValue{}, &IntValue{}, &TypeValue{}}
 

diff --git a/internal/schema/protobuf_dec.go b/internal/schema/protobuf_dec.go
@@ -197,6 +197,14 @@ func metadataToSchema(s *schemapb.Metadata) Metadata {
 			Digest: s.SqlMigration.Digest,
 		}
 
+	case *schemapb.Metadata_Artefact:
+		return &MetadataArtefact{
+			Pos:        PosFromProto(s.Artefact.Pos),
+			Executable: s.Artefact.Executable,
+			Path:       s.Artefact.Path,
+			Digest:     s.Artefact.Digest,
+		}
+
 	default:
 		panic(fmt.Sprintf("unhandled metadata type: %T", s))
 	}

diff --git a/internal/schema/protobuf_enc.go b/internal/schema/protobuf_enc.go
@@ -100,6 +100,9 @@ func metadataListToProto(nodes []Metadata) []*schemapb.Metadata {
 		case *MetadataSQLMigration:
 			v = &schemapb.Metadata_SqlMigration{SqlMigration: n.ToProto().(*schemapb.MetadataSQLMigration)}
 
+		case *MetadataArtefact:
+			v = &schemapb.Metadata_Artefact{Artefact: n.ToProto().(*schemapb.MetadataArtefact)}
+
 		default:
 			panic(fmt.Sprintf("unhandled metadata type %T", n))
 		}

diff --git a/internal/schema/schema_test.go b/internal/schema/schema_test.go
@@ -317,6 +317,8 @@ func TestParsing(t *testing.T) {
 		{name: "TimeEcho",
 			input: `
 				module echo {
+					+artefact "echo" "asf4as1fdfa2"
+
 					data EchoRequest {
 						name String?
 					}
@@ -346,6 +348,9 @@ func TestParsing(t *testing.T) {
 			expected: &Schema{
 				Modules: []*Module{{
 					Name: "echo",
+					Metadata: []Metadata{
+						&MetadataArtefact{Path: "echo", Digest: "asf4as1fdfa2", Executable: false},
+					},
 					Decls: []Decl{
 						&Data{Name: "EchoRequest", Fields: []*Field{{Name: "name", Type: &Optional{Type: &String{}}}}},
 						&Data{Name: "EchoResponse", Fields: []*Field{{Name: "message", Type: &String{}}}},

diff --git a/internal/schema/validate.go b/internal/schema/validate.go
@@ -180,7 +180,7 @@ func ValidateModuleInSchema(schema *Schema, m optional.Option[*Module]) (*Schema
 						validateRetries(module, md, optional.Some(n.Request), scopes, optional.Some(schema))
 
 					case *MetadataCronJob, *MetadataCalls, *MetadataConfig, *MetadataDatabases, *MetadataAlias, *MetadataTypeMap,
-						*MetadataEncoding, *MetadataSecrets, *MetadataPublisher, *MetadataSQLMigration, DatabaseConnector:
+						*MetadataEncoding, *MetadataSecrets, *MetadataPublisher, *MetadataSQLMigration, *MetadataArtefact, DatabaseConnector:
 					}
 				}
 			case *Database:
@@ -224,7 +224,7 @@ func ValidateModuleInSchema(schema *Schema, m optional.Option[*Module]) (*Schema
 				Value, *IntValue, *StringValue, *TypeValue, *Config, *Secret, Symbol, Named,
 				*MetadataSubscriber, *Topic, *MetadataTypeMap, *MetadataEncoding, *MetadataPublisher,
 				*MetadataSQLMigration, *DSNDatabaseConnector, *DatabaseRuntime, DatabaseConnector,
-				*AWSIAMAuthDatabaseConnector, *DatabaseRuntimeConnections:
+				*AWSIAMAuthDatabaseConnector, *DatabaseRuntimeConnections, *MetadataArtefact:
 			}
 			return next()
 		})
@@ -367,7 +367,7 @@ func ValidateModule(module *Module) error {
 			*Unit, *Any, *TypeParameter, *Enum, *EnumVariant, *IntValue, *StringValue, *TypeValue,
 			*Config, *Secret, *MetadataSubscriber, *MetadataTypeMap, *MetadataEncoding, *MetadataPublisher,
 			*MetadataSQLMigration, *DSNDatabaseConnector, *DatabaseRuntime, *AWSIAMAuthDatabaseConnector,
-			DatabaseConnector, *DatabaseRuntimeConnections:
+			*MetadataArtefact, DatabaseConnector, *DatabaseRuntimeConnections:
 
 		case Named, Symbol, Type, Metadata, Value, Decl: // Union types.
 		}
@@ -467,6 +467,8 @@ func sortMetadataType(md Metadata) {
 		sortRefs(m.Topics)
 	case *MetadataSQLMigration:
 		return
+	case *MetadataArtefact:
+		return
 	}
 }
 
@@ -499,6 +501,8 @@ func getMetadataSortingPriority(metadata Metadata) int {
 		priority = 12
 	case *MetadataSQLMigration:
 		priority = 13
+	case *MetadataArtefact:
+		priority = 14
 	}
 	return priority
 }
@@ -697,7 +701,7 @@ func validateVerbMetadata(scopes Scopes, module *Module, n *Verb) (merr []error)
 			subErrs := validateVerbSubscriptions(module, n, md, scopes)
 			merr = append(merr, subErrs...)
 		case *MetadataCalls, *MetadataConfig, *MetadataDatabases, *MetadataAlias, *MetadataTypeMap, *MetadataEncoding,
-			*MetadataSecrets, *MetadataPublisher, *MetadataSQLMigration:
+			*MetadataSecrets, *MetadataPublisher, *MetadataSQLMigration, *MetadataArtefact:
 		}
 	}
 	return

diff --git a/python-runtime/ftl/src/ftl/protos/xyz/block/ftl/schema/v1/schema_pb2.py b/python-runtime/ftl/src/ftl/protos/xyz/block/ftl/schema/v1/schema_pb2.py