Experimental/json serializable columns (#695)

* Attempting to implement the json serializable. First spec passes, but it's super hacky

* round two reverts some of the previous commit. Now using a column converter. Still needs lots more specs

* reverting these changes because they're refactors and not really related to this PR. There's already enough noise.

* use separate branch with json changes and we no longer need any patches

* Adding the ability to query serializable columns

* This line wasn't needed

* Go back to original PG Shard, but use master now that Matthew's PR was merged

db/migrations/20210703234151_add_metadata_to_blobs.cr

@@ -0,0 +1,15 @@
+class AddMetadataToBlobs::V20210703234151 < Avram::Migrator::Migration::V1
+  def migrate
+    alter :blobs do
+      add metadata : JSON::Any, default: JSON::Any.new({} of String => JSON::Any)
+      add media : JSON::Any?, fill_existing_with: :nothing
+    end
+  end
+
+  def rollback
+    alter :blobs do
+      remove :metadata
+      remove :media
+    end
+  end
+end

shard.yml

@@ -18,7 +18,7 @@ dependencies:
     version: ~> 0.1.0
   pg:
     github: will/crystal-pg
-    version: ~> 0.23.0
+    branch: master
   habitat:
     github: luckyframework/habitat
     version: ~> 0.4.7

spec/json_column_spec.cr

@@ -27,13 +27,17 @@ describe "JSON Columns" do
   it "should convert scalars and save forms" do
     form1 = SaveBlob.new
     form1.set_doc_from_param(42)
+    form1.set_metadata_from_param(BlobMetadata.from_json("{}"))
     form1.doc.value.should eq JSON::Any.new(42_i64)
+    form1.metadata.value.should be_a(BlobMetadata)
+
     form1.save!
     blob1 = BlobQuery.new.last
     blob1.doc.should eq JSON::Any.new(42_i64)
 
     form2 = SaveBlob.new
     form2.set_doc_from_param("hey")
+    form2.set_metadata_from_param(BlobMetadata.from_json("{}"))
     form2.doc.value.should eq JSON::Any.new("hey")
     form2.save!
     blob2 = BlobQuery.new.last
@@ -43,13 +47,15 @@ describe "JSON Columns" do
   it "should convert hashes and arrays and save forms" do
     form1 = SaveBlob.new
     form1.set_doc_from_param(%w[a b c])
+    form1.set_metadata_from_param(BlobMetadata.from_json("{}"))
     form1.doc.value.should eq %w[a b c].map { |v| JSON::Any.new(v) }
     form1.save!
     blob1 = BlobQuery.new.last
     blob1.doc.should eq %w[a b c].map { |v| JSON::Any.new(v) }
 
     form2 = SaveBlob.new
     form2.set_doc_from_param({"foo" => {"bar" => "baz"}})
+    form2.set_metadata_from_param(BlobMetadata.from_json("{}"))
     form2.doc.value.should eq JSON::Any.new({
       "foo" => JSON::Any.new({
         "bar" => JSON::Any.new("baz"),
@@ -63,4 +69,25 @@ describe "JSON Columns" do
       }),
     })
   end
+
+  describe "serialized" do
+    it "saves the serialized value" do
+      SaveBlob.create(metadata: BlobMetadata.from_json("{}")) do |operation, blob|
+        operation.saved?.should be_true
+        blob.should_not be_nil
+        blob.not_nil!.metadata.should be_a(BlobMetadata)
+        blob.not_nil!.metadata.name.should be_nil
+        blob.not_nil!.media.should be_nil
+      end
+    end
+
+    it "queries serialized columns" do
+      one = BlobMetadata.from_json({name: "One", code: 4}.to_json)
+      two = BlobMetadata.from_json({name: "Two", code: 9}.to_json)
+      BlobFactory.create &.metadata(one)
+      BlobFactory.create &.metadata(two)
+
+      BlobQuery.new.metadata(two).select_count.should eq(1)
+    end
+  end
 end

spec/queryable_spec.cr

@@ -1031,17 +1031,17 @@ describe Avram::Queryable do
         blob = BlobFactory.new.doc(JSON::Any.new({"foo" => JSON::Any.new("bar")})).create
 
         query = JSONQuery.new.static_foo
-        query.to_sql.should eq ["SELECT blobs.id, blobs.created_at, blobs.updated_at, blobs.doc FROM blobs WHERE blobs.doc = $1", "{\"foo\":\"bar\"}"]
+        query.to_sql.should eq ["SELECT blobs.id, blobs.created_at, blobs.updated_at, blobs.doc, blobs.metadata, blobs.media FROM blobs WHERE blobs.doc = $1", "{\"foo\":\"bar\"}"]
         result = query.first
         result.should eq blob
 
         query2 = JSONQuery.new.foo_with_value("bar")
-        query2.to_sql.should eq ["SELECT blobs.id, blobs.created_at, blobs.updated_at, blobs.doc FROM blobs WHERE blobs.doc = $1", "{\"foo\":\"bar\"}"]
+        query2.to_sql.should eq ["SELECT blobs.id, blobs.created_at, blobs.updated_at, blobs.doc, blobs.metadata, blobs.media FROM blobs WHERE blobs.doc = $1", "{\"foo\":\"bar\"}"]
         result = query2.first
         result.should eq blob
 
         query3 = JSONQuery.new.foo_with_value("baz")
-        query3.to_sql.should eq ["SELECT blobs.id, blobs.created_at, blobs.updated_at, blobs.doc FROM blobs WHERE blobs.doc = $1", "{\"foo\":\"baz\"}"]
+        query3.to_sql.should eq ["SELECT blobs.id, blobs.created_at, blobs.updated_at, blobs.doc, blobs.metadata, blobs.media FROM blobs WHERE blobs.doc = $1", "{\"foo\":\"baz\"}"]
         expect_raises(Avram::RecordNotFoundError) do
           query3.first
         end
@@ -1386,7 +1386,7 @@ describe Avram::Queryable do
       query.to_prepared_sql.should eq(%{SELECT #{Bucket::COLUMN_SQL} FROM buckets WHERE buckets.names = '{"Larry","Moe","Curly"}' AND buckets.numbers = '{1,2,3}'})
 
       query = Blob::BaseQuery.new.doc(JSON::Any.new({"properties" => JSON::Any.new("sold")}))
-      query.to_prepared_sql.should eq(%{SELECT blobs.id, blobs.created_at, blobs.updated_at, blobs.doc FROM blobs WHERE blobs.doc = '{"properties":"sold"}'})
+      query.to_prepared_sql.should eq(%{SELECT blobs.id, blobs.created_at, blobs.updated_at, blobs.doc, blobs.metadata, blobs.media FROM blobs WHERE blobs.doc = '{"properties":"sold"}'})
 
       query = UserQuery.new.name.in(["Don", "Juan"]).age.gt(30)
       query.to_prepared_sql.should eq(%{SELECT #{User::COLUMN_SQL} FROM users WHERE users.name = ANY ('{"Don","Juan"}') AND users.age > '30'})

spec/support/factories/blob_factory.cr

@@ -1,5 +1,7 @@
 class BlobFactory < BaseFactory
   def initialize
     doc JSON::Any.new({"foo" => JSON::Any.new("bar")})
+    metadata(BlobMetadata.from_json({name: "Test", code: 4}.to_json))
+    media nil
   end
 end

spec/support/models/blob.cr

@@ -1,5 +1,20 @@
+class BlobMetadata
+  include JSON::Serializable
+
+  property name : String?
+  property code : Int32?
+end
+
+class MediaMetadata
+  include JSON::Serializable
+
+  property image : String?
+end
+
 class Blob < BaseModel
   table do
     column doc : JSON::Any?
+    column metadata : BlobMetadata, serialize: true
+    column media : MediaMetadata?, serialize: true
   end
 end

src/avram.cr

@@ -2,6 +2,10 @@ require "dexter"
 require "wordsmith"
 require "habitat"
 require "pulsar"
+require "db"
+require "pg"
+require "uuid"
+
 require "./avram/object_extensions"
 require "./avram/criteria"
 require "./avram/type"
@@ -11,9 +15,6 @@ require "./avram/charms/**"
 require "./avram/migrator/**"
 require "./avram/tasks/**"
 require "./avram/**"
-require "db"
-require "pg"
-require "uuid"
 
 module Avram
   Habitat.create do

src/avram/charms/json_extensions.cr

@@ -1,3 +1,38 @@
+module JSON::Serializable
+  macro included
+    def self.adapter
+      Lucky(self)
+    end
+  end
+
+  module Lucky(T)
+    include Avram::Type
+
+    def self.criteria(query : R, column) forall R
+      Criteria(R, T).new(query, column)
+    end
+
+    def from_db!(value)
+      value
+    end
+
+    def parse(value : JSON::Serializable)
+      SuccessfulCast(JSON::Serializable).new value
+    end
+
+    def parse(value)
+      SuccessfulCast(JSON::Serializable).new T.from_json(value)
+    end
+
+    def to_db(value)
+      value.to_json
+    end
+
+    class Criteria(T, V) < Avram::Criteria(T, V)
+    end
+  end
+end
+
 struct JSON::Any
   def self.adapter
     Lucky

src/avram/model.cr

@@ -8,6 +8,13 @@ abstract class Avram::Model
   include Avram::Polymorphic
   include Avram::SchemaEnforcer
 
+  module JSONConverter(T)
+    def self.from_rs(rs : DB::ResultSet)
+      value = rs.read(JSON::PullParser?)
+      T.new(value) if value
+    end
+  end
+
   macro inherited
     COLUMNS = [] of Nil # types are not checked in macros
     ASSOCIATIONS = [] of Nil # types are not checked in macros
@@ -184,12 +191,15 @@ abstract class Avram::Model
         {{column[:name]}}: {
           {% if column[:type].id == Array(Float64).id %}
             type: Array(PG::Numeric),
-          {% elsif column[:type].is_a?(Generic) %}
+          {% elsif column[:type].is_a?(Generic) || column[:serialized] %}
             type: {{column[:type]}},
           {% else %}
             type: {{column[:type]}}::Lucky::ColumnType,
           {% end %}
           nilable: {{column[:nilable]}},
+          {% if column[:serialized] %}
+          converter: JSONConverter({{column[:type]}}),
+          {% end %}
         },
       {% end %}
     })
@@ -232,7 +242,7 @@ abstract class Avram::Model
     {% end %}
   end
 
-  macro column(type_declaration, autogenerated = false)
+  macro column(type_declaration, autogenerated = false, serialize is_serialized = false)
     {% if type_declaration.type.is_a?(Union) %}
       {% data_type = type_declaration.type.types.first %}
       {% nilable = true %}
@@ -245,7 +255,8 @@ abstract class Avram::Model
     {% else %}
       {% value = nil %}
     {% end %}
-    {% COLUMNS << {name: type_declaration.var, type: data_type, nilable: nilable.id, autogenerated: autogenerated, value: value} %}
+
+    {% COLUMNS << {name: type_declaration.var, type: data_type, nilable: nilable.id, autogenerated: autogenerated, value: value, serialized: is_serialized} %}
   end
 
   macro setup_column_info_methods(columns, *args, **named_args)