From eb88a98bcdc54fd016078bb73a06e5650e9c4849 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Wed, 28 Aug 2024 08:19:22 -0300 Subject: [PATCH 1/3] update caching and logging --- core/dbio/connection/connection.go | 6 ++++-- core/dbio/connection/connection_discover.go | 4 ---- core/dbio/connection/connection_local.go | 2 ++ core/sling/replication.go | 13 ++++++------- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/core/dbio/connection/connection.go b/core/dbio/connection/connection.go index dd2ef168..c4b5eccd 100644 --- a/core/dbio/connection/connection.go +++ b/core/dbio/connection/connection.go @@ -264,7 +264,8 @@ func (c *Connection) AsDatabase(cache ...bool) (dc database.Connection, err erro return nil, g.Error("not a database type: %s", c.Type) } - if len(cache) > 0 && cache[0] { + // default cache to true + if len(cache) == 0 || (len(cache) > 0 && cache[0]) { if c.Database == nil { c.Database, err = database.NewConnContext( c.Context().Ctx, c.URL(), g.MapToKVArr(c.DataS())..., @@ -286,7 +287,8 @@ func (c *Connection) AsFile(cache ...bool) (fc filesys.FileSysClient, err error) return nil, g.Error("not a file system type: %s", c.Type) } - if len(cache) > 0 && cache[0] { + // default cache to true + if len(cache) == 0 || (len(cache) > 0 && cache[0]) { if c.File == nil { c.File, err = filesys.NewFileSysClientFromURLContext( c.Context().Ctx, c.URL(), g.MapToKVArr(c.DataS())..., diff --git a/core/dbio/connection/connection_discover.go b/core/dbio/connection/connection_discover.go index e4a35f18..2b4860b2 100644 --- a/core/dbio/connection/connection_discover.go +++ b/core/dbio/connection/connection_discover.go @@ -26,7 +26,6 @@ func (c *Connection) Test() (ok bool, err error) { if err != nil { return ok, g.Error(err, "could not connect to %s", c.Name) } - defer dbConn.Close() case c.Type.IsFile(): fileClient, err := c.AsFile() if err != nil { @@ -36,7 +35,6 @@ func (c *Connection) Test() (ok bool, err error) { if err != nil { return ok, g.Error(err, "could not connect to %s", c.Name) } - defer fileClient.Close() url := c.URL() @@ -97,7 +95,6 @@ func (c *Connection) Discover(opt *DiscoverOptions) (ok bool, nodes filesys.File if err != nil { return ok, nodes, schemata, g.Error(err, "could not connect to %s", c.Name) } - defer dbConn.Close() var table database.Table if opt.Pattern != "" { @@ -142,7 +139,6 @@ func (c *Connection) Discover(opt *DiscoverOptions) (ok bool, nodes filesys.File if err != nil { return ok, nodes, schemata, g.Error(err, "could not connect to %s", c.Name) } - defer fileClient.Close() url := c.URL() if opt.Pattern != "" { diff --git a/core/dbio/connection/connection_local.go b/core/dbio/connection/connection_local.go index b764d0f0..67ec3a10 100644 --- a/core/dbio/connection/connection_local.go +++ b/core/dbio/connection/connection_local.go @@ -277,6 +277,7 @@ func (ec *EnvConns) Discover(name string, opt *DiscoverOptions) (nodes filesys.F return nodes, schemata, g.Error("Invalid Connection name: %s. Make sure it is created. See https://docs.slingdata.io/sling-cli/environment", name) } + defer conn.Connection.Close() _, nodes, schemata, err = conn.Connection.Discover(opt) return } @@ -286,6 +287,7 @@ func (ec *EnvConns) Test(name string) (ok bool, err error) { if !ok1 || name == "" { return ok, g.Error("Invalid Connection name: %s. Make sure it is created. See https://docs.slingdata.io/sling-cli/environment", name) } + defer conn.Connection.Close() ok, err = conn.Connection.Test() return } diff --git a/core/sling/replication.go b/core/sling/replication.go index cecfb75e..e593cd05 100644 --- a/core/sling/replication.go +++ b/core/sling/replication.go @@ -225,6 +225,7 @@ func (rd *ReplicationConfig) ProcessWildcardsDatabase(c connection.Connection, w return g.Error("could not get tables for schema: %s", schemaT.Schema) } + streamsAdded := []string{} for _, table := range schemata.Tables() { // add to stream map @@ -238,8 +239,9 @@ func (rd *ReplicationConfig) ProcessWildcardsDatabase(c connection.Connection, w cfg := rd.Streams[wildcardName] rd.AddStream(table.FullName(), cfg) - g.Debug("wildcard '%s' matched stream => %s", wildcardName, table.FullName()) + streamsAdded = append(streamsAdded, table.FullName()) } + g.Debug("wildcard '%s' matched %d streams => %+v", wildcardName, len(streamsAdded), streamsAdded) // delete * from stream map rd.DeleteStream(wildcardName) @@ -274,7 +276,7 @@ func (rd *ReplicationConfig) ProcessWildcardsFile(c connection.Connection, wildc return g.Error("could not get files for schema: %s", wildcardName) } - added := 0 + streamsAdded := []string{} for _, node := range nodes { streamName, streamConfig, found := rd.GetStream(node.Path()) if found { @@ -295,16 +297,13 @@ func (rd *ReplicationConfig) ProcessWildcardsFile(c connection.Connection, wildc // add path rd.AddStream(node.Path(), rd.Streams[wildcardName]) - added++ - g.Debug("wildcard '%s' matched stream => %s", wildcardName, node.Path()) + streamsAdded = append(streamsAdded, node.Path()) } + g.Debug("wildcard '%s' matched %d streams => %+v", wildcardName, len(streamsAdded), streamsAdded) // delete from stream map rd.DeleteStream(wildcardName) - if added == 0 { - g.Debug("0 streams added for %#v (nodes=%d)", wildcardName, len(nodes)) - } } return From f3c34f76caaeb568c926681c9022d082e38d7bae Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Wed, 28 Aug 2024 11:57:53 -0300 Subject: [PATCH 2/3] add home/sling in Dockerfiles --- cmd/sling/Dockerfile | 2 ++ cmd/sling/Dockerfile.arm64 | 2 ++ 2 files changed, 4 insertions(+) diff --git a/cmd/sling/Dockerfile b/cmd/sling/Dockerfile index d1803c75..e33001d5 100755 --- a/cmd/sling/Dockerfile +++ b/cmd/sling/Dockerfile @@ -27,9 +27,11 @@ COPY --chown=sling:sling sling /usr/local/bin/sling RUN chmod 755 /usr/local/bin/sling # Switch to non-root user +RUN mkdir -p /home/sling && chmod 755 /home/sling && chown sling:sling /home/sling USER sling # Final ENV updates +ENV DBUS_SESSION_BUS_ADDRESS="/dev/null" ENV ORACLE_HOME="/usr/lib/oracle/19.3/client64" ENV LD_LIBRARY_PATH="/usr/lib/oracle/19.3/client64/lib" ENV PATH="${PATH}:/usr/local/go/bin:/root/go/bin:/opt/mssql-tools/bin:/usr/lib/oracle/19.3/client64/bin" diff --git a/cmd/sling/Dockerfile.arm64 b/cmd/sling/Dockerfile.arm64 index f2756088..a3d7835d 100755 --- a/cmd/sling/Dockerfile.arm64 +++ b/cmd/sling/Dockerfile.arm64 @@ -30,9 +30,11 @@ COPY --chown=sling:sling sling /usr/local/bin/sling RUN chmod 755 /usr/local/bin/sling # Switch to non-root user +RUN mkdir -p /home/sling && chmod 755 /home/sling && chown sling:sling /home/sling USER sling # Final ENV updates +ENV DBUS_SESSION_BUS_ADDRESS="/dev/null" ENV ORACLE_HOME="/opt/oracle/instantclient_19_19" ENV LD_LIBRARY_PATH="/opt/oracle/instantclient_19_19" ENV PATH="${PATH}:/usr/local/go/bin:/root/go/bin:/opt/oracle/instantclient_19_19:/opt/mssql-tools18/bin" From 740de17185069582490c144ba294b13010c270c8 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Wed, 28 Aug 2024 11:58:02 -0300 Subject: [PATCH 3/3] improve GetMachineID --- core/store/db.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/store/db.go b/core/store/db.go index e7eab0a7..d04c8ab8 100644 --- a/core/store/db.go +++ b/core/store/db.go @@ -108,6 +108,10 @@ func settings() { } func GetMachineID() string { + if Db == nil { + machineID, _ := machineid.ProtectedID("sling") + return machineID + } s := &Setting{Key: "machine-id"} Db.First(&s) return s.Value