From 0a2992364a680e91fc67906d7776aeef6c7180e8 Mon Sep 17 00:00:00 2001 From: Arthur Date: Sat, 4 Mar 2023 09:57:11 -0500 Subject: [PATCH] WebM (VP8) recorder and Jitter Buffer refactoring - refactored VP8 frame builder and resolved issue with artifacts - refactored Jitter Buffer usage - mirrored ebml-go package (WebM) and patched it to get rid of null termination in encoder --- README.md | 2 + VERSION | 2 +- config/bbb-webrtc-recorder.yml | 1 + go.mod | 32 +- go.sum | 83 +- internal/config/config.go | 18 +- internal/server/server.go | 15 +- internal/server/session.go | 10 +- internal/webrtc/recorder/recorder.go | 15 +- internal/webrtc/recorder/webm.go | 219 +++--- internal/webrtc/utils/jitter_buffer.go | 2 +- internal/webrtc/utils/nack.go | 2 +- internal/webrtc/utils/nack_test.go | 2 +- internal/webrtc/webrtc.go | 201 +++-- pkg/ebml-go/.gitignore | 1 + pkg/ebml-go/172-null-term.patch | 129 ++++ pkg/ebml-go/LICENSE | 202 +++++ pkg/ebml-go/README.md | 30 + pkg/ebml-go/block.go | 168 ++++ pkg/ebml-go/block_test.go | 163 ++++ pkg/ebml-go/codecov.yml | 10 + pkg/ebml-go/datatype.go | 79 ++ pkg/ebml-go/datatype_test.go | 26 + pkg/ebml-go/ebml.go | 19 + pkg/ebml-go/elementtable.go | 283 +++++++ pkg/ebml-go/elementtype.go | 550 ++++++++++++++ pkg/ebml-go/elementtype_test.go | 68 ++ pkg/ebml-go/error.go | 87 +++ pkg/ebml-go/error_test.go | 87 +++ pkg/ebml-go/examples/README.md | 34 + pkg/ebml-go/examples/rtp-to-webm/.gitignore | 2 + pkg/ebml-go/examples/rtp-to-webm/main.go | 140 ++++ .../examples/webm-roundtrip/.gitignore | 2 + pkg/ebml-go/examples/webm-roundtrip/main.go | 56 ++ .../examples/webm-roundtrip/sample.webm | Bin 0 -> 13017 bytes pkg/ebml-go/go.mod | 3 + pkg/ebml-go/hook.go | 59 ++ .../internal/buffercloser/buffercloser.go | 68 ++ .../buffercloser/buffercloser_test.go | 56 ++ pkg/ebml-go/internal/errs/errors_112.go | 41 + pkg/ebml-go/internal/errs/errors_113.go | 28 + pkg/ebml-go/internal/errs/errors_test.go | 83 ++ pkg/ebml-go/lacer.go | 166 ++++ pkg/ebml-go/lacer_test.go | 187 +++++ pkg/ebml-go/marshal.go | 328 ++++++++ pkg/ebml-go/marshal_roundtrip_test.go | 113 +++ pkg/ebml-go/marshal_test.go | 715 ++++++++++++++++++ pkg/ebml-go/matroska_official_test.go | 120 +++ pkg/ebml-go/mkv/const.go | 34 + pkg/ebml-go/mkv/mkv.go | 42 + pkg/ebml-go/mkvcore/blockreader.go | 171 +++++ pkg/ebml-go/mkvcore/blockreader_test.go | 440 +++++++++++ pkg/ebml-go/mkvcore/blockwriter.go | 249 ++++++ pkg/ebml-go/mkvcore/blockwriter_test.go | 582 ++++++++++++++ pkg/ebml-go/mkvcore/framebuf.go | 53 ++ pkg/ebml-go/mkvcore/framebuf_test.go | 63 ++ pkg/ebml-go/mkvcore/interceptor.go | 237 ++++++ pkg/ebml-go/mkvcore/interceptor_test.go | 330 ++++++++ pkg/ebml-go/mkvcore/interface.go | 58 ++ pkg/ebml-go/mkvcore/mkvcore.go | 18 + pkg/ebml-go/mkvcore/option.go | 165 ++++ pkg/ebml-go/mkvcore/seekhead.go | 61 ++ pkg/ebml-go/mkvcore/sizedwriter.go | 41 + pkg/ebml-go/mkvcore/sizedwriter_test.go | 50 ++ pkg/ebml-go/mkvcore/struct.go | 76 ++ pkg/ebml-go/reader.go | 79 ++ pkg/ebml-go/reader_test.go | 103 +++ pkg/ebml-go/tag.go | 83 ++ pkg/ebml-go/tag_test.go | 97 +++ pkg/ebml-go/testutils_test.go | 35 + pkg/ebml-go/unlacer.go | 173 +++++ pkg/ebml-go/unlacer_test.go | 208 +++++ pkg/ebml-go/unmarshal.go | 318 ++++++++ pkg/ebml-go/unmarshal_test.go | 688 +++++++++++++++++ pkg/ebml-go/value.go | 482 ++++++++++++ pkg/ebml-go/value_test.go | 429 +++++++++++ pkg/ebml-go/webm/blockwriter.go | 70 ++ pkg/ebml-go/webm/blockwriter_test.go | 194 +++++ pkg/ebml-go/webm/const.go | 40 + pkg/ebml-go/webm/interface.go | 52 ++ pkg/ebml-go/webm/webm.go | 137 ++++ web/public/index.html | 66 +- 82 files changed, 10325 insertions(+), 306 deletions(-) create mode 100644 pkg/ebml-go/.gitignore create mode 100644 pkg/ebml-go/172-null-term.patch create mode 100644 pkg/ebml-go/LICENSE create mode 100644 pkg/ebml-go/README.md create mode 100644 pkg/ebml-go/block.go create mode 100644 pkg/ebml-go/block_test.go create mode 100644 pkg/ebml-go/codecov.yml create mode 100644 pkg/ebml-go/datatype.go create mode 100644 pkg/ebml-go/datatype_test.go create mode 100644 pkg/ebml-go/ebml.go create mode 100644 pkg/ebml-go/elementtable.go create mode 100644 pkg/ebml-go/elementtype.go create mode 100644 pkg/ebml-go/elementtype_test.go create mode 100644 pkg/ebml-go/error.go create mode 100644 pkg/ebml-go/error_test.go create mode 100644 pkg/ebml-go/examples/README.md create mode 100644 pkg/ebml-go/examples/rtp-to-webm/.gitignore create mode 100644 pkg/ebml-go/examples/rtp-to-webm/main.go create mode 100644 pkg/ebml-go/examples/webm-roundtrip/.gitignore create mode 100644 pkg/ebml-go/examples/webm-roundtrip/main.go create mode 100644 pkg/ebml-go/examples/webm-roundtrip/sample.webm create mode 100644 pkg/ebml-go/go.mod create mode 100644 pkg/ebml-go/hook.go create mode 100644 pkg/ebml-go/internal/buffercloser/buffercloser.go create mode 100644 pkg/ebml-go/internal/buffercloser/buffercloser_test.go create mode 100644 pkg/ebml-go/internal/errs/errors_112.go create mode 100644 pkg/ebml-go/internal/errs/errors_113.go create mode 100644 pkg/ebml-go/internal/errs/errors_test.go create mode 100644 pkg/ebml-go/lacer.go create mode 100644 pkg/ebml-go/lacer_test.go create mode 100644 pkg/ebml-go/marshal.go create mode 100644 pkg/ebml-go/marshal_roundtrip_test.go create mode 100644 pkg/ebml-go/marshal_test.go create mode 100644 pkg/ebml-go/matroska_official_test.go create mode 100644 pkg/ebml-go/mkv/const.go create mode 100644 pkg/ebml-go/mkv/mkv.go create mode 100644 pkg/ebml-go/mkvcore/blockreader.go create mode 100644 pkg/ebml-go/mkvcore/blockreader_test.go create mode 100644 pkg/ebml-go/mkvcore/blockwriter.go create mode 100644 pkg/ebml-go/mkvcore/blockwriter_test.go create mode 100644 pkg/ebml-go/mkvcore/framebuf.go create mode 100644 pkg/ebml-go/mkvcore/framebuf_test.go create mode 100644 pkg/ebml-go/mkvcore/interceptor.go create mode 100644 pkg/ebml-go/mkvcore/interceptor_test.go create mode 100644 pkg/ebml-go/mkvcore/interface.go create mode 100644 pkg/ebml-go/mkvcore/mkvcore.go create mode 100644 pkg/ebml-go/mkvcore/option.go create mode 100644 pkg/ebml-go/mkvcore/seekhead.go create mode 100644 pkg/ebml-go/mkvcore/sizedwriter.go create mode 100644 pkg/ebml-go/mkvcore/sizedwriter_test.go create mode 100644 pkg/ebml-go/mkvcore/struct.go create mode 100644 pkg/ebml-go/reader.go create mode 100644 pkg/ebml-go/reader_test.go create mode 100644 pkg/ebml-go/tag.go create mode 100644 pkg/ebml-go/tag_test.go create mode 100644 pkg/ebml-go/testutils_test.go create mode 100644 pkg/ebml-go/unlacer.go create mode 100644 pkg/ebml-go/unlacer_test.go create mode 100644 pkg/ebml-go/unmarshal.go create mode 100644 pkg/ebml-go/unmarshal_test.go create mode 100644 pkg/ebml-go/value.go create mode 100644 pkg/ebml-go/value_test.go create mode 100644 pkg/ebml-go/webm/blockwriter.go create mode 100644 pkg/ebml-go/webm/blockwriter_test.go create mode 100644 pkg/ebml-go/webm/const.go create mode 100644 pkg/ebml-go/webm/interface.go create mode 100644 pkg/ebml-go/webm/webm.go diff --git a/README.md b/README.md index 2fd36ba..ffdbbec 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,8 @@ webrtc: # UDP port range to be used rtcMinPort: 24577 rtcMaxPort: 32768 + # Jitter Buffer size + jitterBuffer: 512 # List of IceServers used for RTC iceServers: - urls: diff --git a/VERSION b/VERSION index 8f7a2d1..2aef3b2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.1 1 \ No newline at end of file +0.2 1 \ No newline at end of file diff --git a/config/bbb-webrtc-recorder.yml b/config/bbb-webrtc-recorder.yml index 2669ea0..ce39b4d 100644 --- a/config/bbb-webrtc-recorder.yml +++ b/config/bbb-webrtc-recorder.yml @@ -17,6 +17,7 @@ pubsub: webrtc: rtcMinPort: 24577 rtcMaxPort: 32768 + jitterBuffer: 512 iceServers: - urls: - stun:stun.l.google.com:19302 diff --git a/go.mod b/go.mod index 69e6c45..4e7bea1 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/bigbluebutton/bbb-webrtc-recorder go 1.19 +replace github.com/at-wat/ebml-go => ./pkg/ebml-go + require ( github.com/AlekSi/pointer v1.2.0 github.com/at-wat/ebml-go v0.16.0 @@ -9,9 +11,11 @@ require ( github.com/gomodule/redigo v1.8.9 github.com/kr/pretty v0.1.0 github.com/mitchellh/mapstructure v1.5.0 + github.com/pion/interceptor v0.1.12 github.com/pion/rtcp v1.2.10 github.com/pion/rtp v1.7.13 - github.com/pion/webrtc/v3 v3.1.50 + github.com/pion/sdp/v3 v3.0.6 + github.com/pion/webrtc/v3 v3.1.56 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.0 github.com/spf13/pflag v1.0.5 @@ -24,21 +28,19 @@ require ( github.com/google/uuid v1.3.0 // indirect github.com/kr/text v0.1.0 // indirect github.com/pion/datachannel v1.5.5 // indirect - github.com/pion/dtls/v2 v2.1.5 // indirect - github.com/pion/ice/v2 v2.2.12 // indirect - github.com/pion/interceptor v0.1.11 // indirect + github.com/pion/dtls/v2 v2.2.6 // indirect + github.com/pion/ice/v2 v2.3.1 // indirect github.com/pion/logging v0.2.2 // indirect - github.com/pion/mdns v0.0.5 // indirect + github.com/pion/mdns v0.0.7 // indirect github.com/pion/randutil v0.1.0 // indirect - github.com/pion/sctp v1.8.5 // indirect - github.com/pion/sdp/v3 v3.0.6 // indirect - github.com/pion/srtp/v2 v2.0.10 // indirect - github.com/pion/stun v0.3.5 // indirect - github.com/pion/transport v0.14.1 // indirect - github.com/pion/turn/v2 v2.0.8 // indirect - github.com/pion/udp v0.1.1 // indirect - golang.org/x/crypto v0.0.0-20221010152910-d6f0a8c073c2 // indirect - golang.org/x/net v0.3.0 // indirect - golang.org/x/sys v0.3.0 // indirect + github.com/pion/sctp v1.8.6 // indirect + github.com/pion/srtp/v2 v2.0.12 // indirect + github.com/pion/stun v0.4.0 // indirect + github.com/pion/transport/v2 v2.0.2 // indirect + github.com/pion/turn/v2 v2.1.0 // indirect + github.com/pion/udp/v2 v2.0.1 // indirect + golang.org/x/crypto v0.6.0 // indirect + golang.org/x/net v0.7.0 // indirect + golang.org/x/sys v0.5.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index d72f6c1..ceb9ef8 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,6 @@ github.com/AlekSi/pointer v1.2.0 h1:glcy/gc4h8HnG2Z3ZECSzZ1IX1x2JxRVuDzaJwQE0+w= github.com/AlekSi/pointer v1.2.0/go.mod h1:gZGfd3dpW4vEc/UlyfKKi1roIqcCgwOIvb0tSNSBle0= github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I= github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/at-wat/ebml-go v0.16.0 h1:3NPy83uMzVRHWdWlcJYSXWn/+u2GmtPQSL1LZJbjcCw= -github.com/at-wat/ebml-go v0.16.0/go.mod h1:w1cJs7zmGsb5nnSvhWGKLCxvfu4FVx5ERvYDIalj1ww= github.com/crazy-max/gonfig v0.6.0 h1:CwP+x1gfor4Ze8lLQKkZw/DOQEMVgwBcoocTYLo3c4A= github.com/crazy-max/gonfig v0.6.0/go.mod h1:Xj0XEnuoHd4OegWtnmdOP6CDNdP9emiwH7IrBM/ka1I= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -49,42 +47,42 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/pion/datachannel v1.5.5 h1:10ef4kwdjije+M9d7Xm9im2Y3O6A6ccQb0zcqZcJew8= github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V118yimL0= -github.com/pion/dtls/v2 v2.1.5 h1:jlh2vtIyUBShchoTDqpCCqiYCyRFJ/lvf/gQ8TALs+c= -github.com/pion/dtls/v2 v2.1.5/go.mod h1:BqCE7xPZbPSubGasRoDFJeTsyJtdD1FanJYL0JGheqY= -github.com/pion/ice/v2 v2.2.12 h1:n3M3lUMKQM5IoofhJo73D3qVla+mJN2nVvbSPq32Nig= -github.com/pion/ice/v2 v2.2.12/go.mod h1:z2KXVFyRkmjetRlaVRgjO9U3ShKwzhlUylvxKfHfd5A= -github.com/pion/interceptor v0.1.11 h1:00U6OlqxA3FFB50HSg25J/8cWi7P6FbSzw4eFn24Bvs= -github.com/pion/interceptor v0.1.11/go.mod h1:tbtKjZY14awXd7Bq0mmWvgtHB5MDaRN7HV3OZ/uy7s8= +github.com/pion/dtls/v2 v2.2.6 h1:yXMxKr0Skd+Ub6A8UqXTRLSywskx93ooMRHsQUtd+Z4= +github.com/pion/dtls/v2 v2.2.6/go.mod h1:t8fWJCIquY5rlQZwA2yWxUS1+OCrAdXrhVKXB5oD/wY= +github.com/pion/ice/v2 v2.3.1 h1:FQCmUfZe2Jpe7LYStVBOP6z1DiSzbIateih3TztgTjc= +github.com/pion/ice/v2 v2.3.1/go.mod h1:aq2kc6MtYNcn4XmMhobAv6hTNJiHzvD0yXRz80+bnP8= +github.com/pion/interceptor v0.1.12 h1:CslaNriCFUItiXS5o+hh5lpL0t0ytQkFnUcbbCs2Zq8= +github.com/pion/interceptor v0.1.12/go.mod h1:bDtgAD9dRkBZpWHGKaoKb42FhDHTG2rX8Ii9LRALLVA= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= -github.com/pion/mdns v0.0.5 h1:Q2oj/JB3NqfzY9xGZ1fPzZzK7sDSD8rZPOvcIQ10BCw= -github.com/pion/mdns v0.0.5/go.mod h1:UgssrvdD3mxpi8tMxAXbsppL3vJ4Jipw1mTCW+al01g= +github.com/pion/mdns v0.0.7 h1:P0UB4Sr6xDWEox0kTVxF0LmQihtCbSAdW0H2nEgkA3U= +github.com/pion/mdns v0.0.7/go.mod h1:4iP2UbeFhLI/vWju/bw6ZfwjJzk0z8DNValjGxR/dD8= github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= -github.com/pion/rtcp v1.2.9/go.mod h1:qVPhiCzAm4D/rxb6XzKeyZiQK69yJpbUDJSF7TgrqNo= github.com/pion/rtcp v1.2.10 h1:nkr3uj+8Sp97zyItdN60tE/S6vk4al5CPRR6Gejsdjc= github.com/pion/rtcp v1.2.10/go.mod h1:ztfEwXZNLGyF1oQDttz/ZKIBaeeg/oWbRYqzBM9TL1I= github.com/pion/rtp v1.7.13 h1:qcHwlmtiI50t1XivvoawdCGTP4Uiypzfrsap+bijcoA= github.com/pion/rtp v1.7.13/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko= -github.com/pion/sctp v1.8.5 h1:JCc25nghnXWOlSn3OVtEnA9PjQ2JsxQbG+CXZ1UkJKQ= github.com/pion/sctp v1.8.5/go.mod h1:SUFFfDpViyKejTAdwD1d/HQsCu+V/40cCs2nZIvC3s0= +github.com/pion/sctp v1.8.6 h1:CUex11Vkt9YS++VhLf8b55O3VqKrWL6W3SDwX4jAqsI= +github.com/pion/sctp v1.8.6/go.mod h1:SUFFfDpViyKejTAdwD1d/HQsCu+V/40cCs2nZIvC3s0= github.com/pion/sdp/v3 v3.0.6 h1:WuDLhtuFUUVpTfus9ILC4HRyHsW6TdugjEX/QY9OiUw= github.com/pion/sdp/v3 v3.0.6/go.mod h1:iiFWFpQO8Fy3S5ldclBkpXqmWy02ns78NOKoLLL0YQw= -github.com/pion/srtp/v2 v2.0.10 h1:b8ZvEuI+mrL8hbr/f1YiJFB34UMrOac3R3N1yq2UN0w= -github.com/pion/srtp/v2 v2.0.10/go.mod h1:XEeSWaK9PfuMs7zxXyiN252AHPbH12NX5q/CFDWtUuA= -github.com/pion/stun v0.3.5 h1:uLUCBCkQby4S1cf6CGuR9QrVOKcvUwFeemaC865QHDg= -github.com/pion/stun v0.3.5/go.mod h1:gDMim+47EeEtfWogA37n6qXZS88L5V6LqFcf+DZA2UA= -github.com/pion/transport v0.12.2/go.mod h1:N3+vZQD9HlDP5GWkZ85LohxNsDcNgofQmyL6ojX5d8Q= -github.com/pion/transport v0.13.0/go.mod h1:yxm9uXpK9bpBBWkITk13cLo1y5/ur5VQpG22ny6EP7g= -github.com/pion/transport v0.13.1/go.mod h1:EBxbqzyv+ZrmDb82XswEE0BjfQFtuw1Nu6sjnjWCsGg= +github.com/pion/srtp/v2 v2.0.12 h1:WrmiVCubGMOAObBU1vwWjG0H3VSyQHawKeer2PVA5rY= +github.com/pion/srtp/v2 v2.0.12/go.mod h1:C3Ep44hlOo2qEYaq4ddsmK5dL63eLehXFbHaZ9F5V9Y= +github.com/pion/stun v0.4.0 h1:vgRrbBE2htWHy7l3Zsxckk7rkjnjOsSM7PHZnBwo8rk= +github.com/pion/stun v0.4.0/go.mod h1:QPsh1/SbXASntw3zkkrIk3ZJVKz4saBY2G7S10P3wCw= github.com/pion/transport v0.14.1 h1:XSM6olwW+o8J4SCmOBb/BpwZypkHeyM0PGFCxNQBr40= github.com/pion/transport v0.14.1/go.mod h1:4tGmbk00NeYA3rUa9+n+dzCCoKkcy3YlYb99Jn2fNnI= -github.com/pion/turn/v2 v2.0.8 h1:KEstL92OUN3k5k8qxsXHpr7WWfrdp7iJZHx99ud8muw= -github.com/pion/turn/v2 v2.0.8/go.mod h1:+y7xl719J8bAEVpSXBXvTxStjJv3hbz9YFflvkpcGPw= -github.com/pion/udp v0.1.1 h1:8UAPvyqmsxK8oOjloDk4wUt63TzFe9WEJkg5lChlj7o= -github.com/pion/udp v0.1.1/go.mod h1:6AFo+CMdKQm7UiA0eUPA8/eVCTx8jBIITLZHc9DWX5M= -github.com/pion/webrtc/v3 v3.1.50 h1:wLMo1+re4WMZ9Kun9qcGcY+XoHkE3i0CXrrc0sjhVCk= -github.com/pion/webrtc/v3 v3.1.50/go.mod h1:y9n09weIXB+sjb9mi0GBBewNxo4TKUQm5qdtT5v3/X4= +github.com/pion/transport/v2 v2.0.0/go.mod h1:HS2MEBJTwD+1ZI2eSXSvHJx/HnzQqRy2/LXxt6eVMHc= +github.com/pion/transport/v2 v2.0.2 h1:St+8o+1PEzPT51O9bv+tH/KYYLMNR5Vwm5Z3Qkjsywg= +github.com/pion/transport/v2 v2.0.2/go.mod h1:vrz6bUbFr/cjdwbnxq8OdDDzHf7JJfGsIRkxfpZoTA0= +github.com/pion/turn/v2 v2.1.0 h1:5wGHSgGhJhP/RpabkUb/T9PdsAjkGLS6toYz5HNzoSI= +github.com/pion/turn/v2 v2.1.0/go.mod h1:yrT5XbXSGX1VFSF31A3c1kCNB5bBZgk/uu5LET162qs= +github.com/pion/udp/v2 v2.0.1 h1:xP0z6WNux1zWEjhC7onRA3EwwSliXqu1ElUZAQhUP54= +github.com/pion/udp/v2 v2.0.1/go.mod h1:B7uvTMP00lzWdyMr/1PVZXtV3wpPIxBRd4Wl6AksXn8= +github.com/pion/webrtc/v3 v3.1.56 h1:ScaiqKQN3liQwT+kJwOBaYP6TwSfixzdUnZmzHAo0a0= +github.com/pion/webrtc/v3 v3.1.56/go.mod h1:7VhbA6ihqJlz6R/INHjyh1b8HpiV9Ct4UQvE1OB/xoM= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -99,7 +97,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -113,9 +110,9 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20221010152910-d6f0a8c073c2 h1:x8vtB3zMecnlqZIwJNUUpwYKYSqCz5jXbiyv0ZJJZeI= -golang.org/x/crypto v0.0.0-20221010152910-d6f0a8c073c2/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= +golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -123,18 +120,14 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201201195509-5d6afe98e0b7/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211201190559-0a0e4e1bb54c/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220531201128-c960675eff93/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.3.0 h1:VWL6FNY2bEEmsGVKabSlHu5Irp34xmMRoqb/9lF9lxk= -golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -151,27 +144,27 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= -golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= diff --git a/internal/config/config.go b/internal/config/config.go index 2a686f3..a7393a8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -44,13 +44,16 @@ func (cfg *Config) SetDefaults() { cfg.PubSub.Adapter = "redis" cfg.PubSub.Adapters = make(map[string]interface{}) cfg.PubSub.Adapters["redis"] = &Redis{ - Address: ":6379", - Network: "tcp", + Address: ":6379", + Network: "tcp", Password: "", } cfg.WebRTC.ICEServers = append(cfg.WebRTC.ICEServers, webrtc.ICEServer{ URLs: []string{"stun:stun.l.google.com:19302"}, }) + cfg.WebRTC.RTCMinPort = 24577 + cfg.WebRTC.RTCMaxPort = 32768 + cfg.WebRTC.JitterBuffer = 512 cfg.HTTP = HTTP{ Enable: false, Port: 8080, @@ -62,8 +65,8 @@ type Recorder struct { } type Redis struct { - Address string `yaml:"address,omitempty"` - Network string `yaml:"network,omitempty"` + Address string `yaml:"address,omitempty"` + Network string `yaml:"network,omitempty"` Password string `yaml:"password,omitempty"` } @@ -79,9 +82,10 @@ type Channels struct { } type WebRTC struct { - ICEServers []webrtc.ICEServer `yaml:"iceServers,omitempty"` - RTCMinPort uint16 `yaml:"rtcMinPort,omitempty"` - RTCMaxPort uint16 `yaml:"rtcMaxPort,omitempty"` + ICEServers []webrtc.ICEServer `yaml:"iceServers,omitempty"` + RTCMinPort uint16 `yaml:"rtcMinPort,omitempty"` + RTCMaxPort uint16 `yaml:"rtcMaxPort,omitempty"` + JitterBuffer uint16 `yaml:"jitterBuffer,omitempty"` } type HTTP struct { diff --git a/internal/server/server.go b/internal/server/server.go index 3b2281d..1276547 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -50,21 +50,8 @@ func (s *Server) HandlePubSub(ctx context.Context, msg []byte) { return } - flowCallbackFn := func() recorder.FlowCallbackFn { - return func(isFlowing bool, keyframeSequence int64, videoTimestamp time.Duration, closed bool) { - var message interface{} - if !closed { - message = events.NewRecordingRtpStatusChanged(e.SessionId, isFlowing, videoTimestamp/time.Millisecond) - } else { - s.CloseSession(e.SessionId) - message = events.NewRecordingStopped(e.SessionId, "closed", videoTimestamp/time.Millisecond) - } - s.PublishPubSub(message) - } - } - var sdp string - if rec, err := recorder.NewRecorder(ctx, s.cfg.Recorder, e.FileName, flowCallbackFn()); err != nil { + if rec, err := recorder.NewRecorder(ctx, s.cfg.Recorder, e.FileName); err != nil { log.WithField("session", ctx.Value("session")). Error(err) s.PublishPubSub(e.Fail(err)) diff --git a/internal/server/session.go b/internal/server/session.go index 98070cc..af1b654 100644 --- a/internal/server/session.go +++ b/internal/server/session.go @@ -39,12 +39,20 @@ func (s *Session) StartRecording(sdp string) string { s.server.CloseSession(s.id) } } + }, func(isFlowing bool, videoTimestamp time.Duration, closed bool) { + var message interface{} + if !closed { + message = events.NewRecordingRtpStatusChanged(s.id, isFlowing, videoTimestamp/time.Millisecond) + } else { + s.server.CloseSession(s.id) + message = events.NewRecordingStopped(s.id, "closed", videoTimestamp/time.Millisecond) + } + s.server.PublishPubSub(message) }) return signal.Encode(answer) } func (s *Session) StopRecording() time.Duration { - //s.webrtc.Close() if !s.stopped { s.stopped = true return s.recorder.Close() diff --git a/internal/webrtc/recorder/recorder.go b/internal/webrtc/recorder/recorder.go index f105652..dfe6326 100644 --- a/internal/webrtc/recorder/recorder.go +++ b/internal/webrtc/recorder/recorder.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/bigbluebutton/bbb-webrtc-recorder/internal/config" "github.com/pion/rtp" - "github.com/pion/webrtc/v3" log "github.com/sirupsen/logrus" "os" "path" @@ -13,16 +12,16 @@ import ( "time" ) -type FlowCallbackFn func(isFlowing bool, keyframeSequence int64, videoTimestamp time.Duration, closed bool) - type Recorder interface { GetFilePath() string - Push(rtp *rtp.Packet, track *webrtc.TrackRemote) - SetContext(ctx context.Context) + PushVideo(rtp *rtp.Packet) + PushAudio(rtp *rtp.Packet) + WithContext(ctx context.Context) + VideoTimestamp() time.Duration Close() time.Duration } -func NewRecorder(ctx context.Context, cfg config.Recorder, file string, fn FlowCallbackFn) (Recorder, error) { +func NewRecorder(ctx context.Context, cfg config.Recorder, file string) (Recorder, error) { ext := filepath.Ext(file) dir := path.Clean(cfg.Directory) @@ -59,8 +58,8 @@ func NewRecorder(ctx context.Context, cfg config.Recorder, file string, fn FlowC var err error switch ext { case ".webm": - r = NewWebmRecorder(file, fn) - r.SetContext(ctx) + r = NewWebmRecorder(file) + r.WithContext(ctx) default: err = fmt.Errorf("unsupported format '%s'", ext) } diff --git a/internal/webrtc/recorder/webm.go b/internal/webrtc/recorder/webm.go index bfb0772..1b7f8fd 100644 --- a/internal/webrtc/recorder/webm.go +++ b/internal/webrtc/recorder/webm.go @@ -5,10 +5,8 @@ import ( "github.com/at-wat/ebml-go/mkvcore" "github.com/at-wat/ebml-go/webm" "github.com/bigbluebutton/bbb-webrtc-recorder/internal" - "github.com/bigbluebutton/bbb-webrtc-recorder/internal/webrtc/utils" "github.com/pion/rtp" "github.com/pion/rtp/codecs" - "github.com/pion/webrtc/v3" "github.com/pion/webrtc/v3/pkg/media/samplebuilder" log "github.com/sirupsen/logrus" "os" @@ -17,6 +15,11 @@ import ( var _ Recorder = (*WebmRecorder)(nil) +const ( + vp8SampleRate = 90000 + secondToNanoseconds = 1000000000 +) + type WebmRecorder struct { ctx context.Context file string @@ -25,48 +28,20 @@ type WebmRecorder struct { audioBuilder, videoBuilder *samplebuilder.SampleBuilder audioTimestamp, videoTimestamp time.Duration - jitterBuffer *utils.JitterBuffer - seqUnwrapper *utils.SequenceUnwrapper - started bool - flowing bool closed bool - flowCallbackFn FlowCallbackFn - keyframeSequence int64 - - statusTicker *time.Ticker - statusTickerChan chan bool + seenKeyFrame bool + currentFrame []byte + packetTimestamp uint32 } -func NewWebmRecorder(file string, fn FlowCallbackFn) *WebmRecorder { +func NewWebmRecorder(file string) *WebmRecorder { r := &WebmRecorder{ - ctx: context.Background(), - file: file, - audioBuilder: samplebuilder.New(10, &codecs.OpusPacket{}, 48000), - videoBuilder: samplebuilder.New(10, &codecs.VP8Packet{}, 90000), - seqUnwrapper: utils.NewSequenceUnwrapper(16), - jitterBuffer: utils.NewJitterBuffer(512), - flowCallbackFn: fn, - } - - r.statusTicker = time.NewTicker(1000 * time.Millisecond) - r.statusTickerChan = make(chan bool) - go func() { - var ts time.Duration - for { - select { - case <-r.statusTickerChan: - return - case <-r.statusTicker.C: - if ts == r.videoTimestamp && r.flowing { - r.flowing = false - r.flowCallbackFn(r.flowing, r.keyframeSequence, r.videoTimestamp, r.closed) - } - } - ts = r.videoTimestamp - } - }() + ctx: context.Background(), + file: file, + audioBuilder: samplebuilder.New(10, &codecs.OpusPacket{}, 48000), + } return r } @@ -75,38 +50,31 @@ func (r *WebmRecorder) GetFilePath() string { return r.file } -func (r *WebmRecorder) SetContext(ctx context.Context) { +func (r *WebmRecorder) WithContext(ctx context.Context) { r.ctx = ctx } -func (r *WebmRecorder) Push(p *rtp.Packet, track *webrtc.TrackRemote) { - if r.closed { - return - } +func (r *WebmRecorder) VideoTimestamp() time.Duration { + return r.videoTimestamp +} - seq := r.seqUnwrapper.Unwrap(uint64(p.SequenceNumber)) - if !r.jitterBuffer.Add(seq, p) { - return - } +func (r *WebmRecorder) AudioTimestamp() time.Duration { + return r.audioTimestamp +} - for _, p := range r.jitterBuffer.NextPackets() { - switch track.Kind() { - case webrtc.RTPCodecTypeAudio: - r.pushOpus(p) - case webrtc.RTPCodecTypeVideo: - r.pushVP8(p) - } - } +func (r *WebmRecorder) PushVideo(p *rtp.Packet) { + r.pushVP8(p) +} + +func (r *WebmRecorder) PushAudio(p *rtp.Packet) { + r.pushOpus(p) } func (r *WebmRecorder) Close() time.Duration { if r.closed { return r.videoTimestamp } - r.flowing = false r.closed = true - r.statusTicker.Stop() - r.statusTickerChan <- true if r.audioWriter != nil { if err := r.audioWriter.Close(); err != nil { @@ -119,7 +87,6 @@ func (r *WebmRecorder) Close() time.Duration { } } if r.started { - //r.flowCallbackFn(r.flowing, r.keyframeSequence, r.videoTimestamp, r.closed) log.WithField("session", r.ctx.Value("session")). Printf("webm writer closed: %s", r.file) } else { @@ -129,8 +96,8 @@ func (r *WebmRecorder) Close() time.Duration { return r.videoTimestamp } -func (r *WebmRecorder) pushOpus(rtpPacket *rtp.Packet) { - r.audioBuilder.Push(rtpPacket) +func (r *WebmRecorder) pushOpus(p *rtp.Packet) { + r.audioBuilder.Push(p) for { sample := r.audioBuilder.Pop() @@ -146,57 +113,53 @@ func (r *WebmRecorder) pushOpus(rtpPacket *rtp.Packet) { } } -func (r *WebmRecorder) pushVP8(rtpPacket *rtp.Packet) { - r.videoBuilder.Push(rtpPacket) +func (r *WebmRecorder) pushVP8(p *rtp.Packet) { + if len(p.Payload) == 0 || r.closed { + return + } - var ts time.Duration - for { - flowing := r.flowing - sample := r.videoBuilder.Pop() - if sample == nil { - return - } - // Read VP8 header. - videoKeyframe := sample.Data[0]&0x1 == 0 - if videoKeyframe { - // jitter buffer content will be dropped up until the keyframe - seq := r.seqUnwrapper.Unwrap(uint64(rtpPacket.SequenceNumber)) - r.jitterBuffer.SetNextPacketsStart(seq) - r.keyframeSequence = seq - - //log.Debug("keyframe #", seq) - - // Keyframe has frame information. - raw := uint(sample.Data[6]) | uint(sample.Data[7])<<8 | uint(sample.Data[8])<<16 | uint(sample.Data[9])<<24 - width := int(raw & 0x3FFF) - height := int((raw >> 16) & 0x3FFF) - - if r.videoWriter == nil || r.audioWriter == nil { - //Initialize WebM saver using received frame size. - r.initWriter(width, height) - } - } - if r.videoWriter != nil { - r.videoTimestamp += sample.Duration - if r.closed { - break - } - if _, err := r.videoWriter.Write(videoKeyframe, int64(r.videoTimestamp/time.Millisecond), sample.Data); err != nil { - panic(err) - } - } + vp8Packet := codecs.VP8Packet{} + if _, err := vp8Packet.Unmarshal(p.Payload); err != nil { + return + } - if ts == r.videoTimestamp { - r.flowing = false - } else { - ts = r.videoTimestamp - r.flowing = true - } - if r.flowing != flowing { - r.flowCallbackFn(r.flowing, r.keyframeSequence, r.videoTimestamp, r.closed) - } + isKeyFrame := vp8Packet.Payload[0]&0x01 == 0 + + switch { + case !r.seenKeyFrame && !isKeyFrame: + return + case r.currentFrame == nil && vp8Packet.S != 1: + return + } + if !r.seenKeyFrame { + r.packetTimestamp = p.Timestamp + } + + r.seenKeyFrame = true + r.currentFrame = append(r.currentFrame, vp8Packet.Payload[0:]...) + + if !p.Marker || len(r.currentFrame) == 0 { + return + } + + // from github.com/pion/webrtc/v3@v3.1.56/pkg/media/samplebuilder/samplebuilder.go#264 + duration := time.Duration((float64(p.Timestamp-r.packetTimestamp)/float64(vp8SampleRate))*secondToNanoseconds) * time.Nanosecond + if r.videoWriter == nil || r.audioWriter == nil { + raw := uint(r.currentFrame[6]) | uint(r.currentFrame[7])<<8 | uint(r.currentFrame[8])<<16 | uint(r.currentFrame[9])<<24 + width := int(raw & 0x3FFF) + height := int((raw >> 16) & 0x3FFF) + r.initWriter(width, height) + } + if r.videoWriter != nil { + r.videoTimestamp += duration + if _, err := r.videoWriter.Write(isKeyFrame, int64(r.videoTimestamp/time.Millisecond), r.currentFrame); err != nil { + log.Error(err) + } } + + r.currentFrame = nil + r.packetTimestamp = p.Timestamp } func (r *WebmRecorder) initWriter(width, height int) { @@ -214,37 +177,35 @@ func (r *WebmRecorder) initWriter(width, height int) { ws, err := webm.NewSimpleBlockWriter(w, []webm.TrackEntry{ { - Name: "Audio", - TrackNumber: 1, - TrackUID: 12345, - CodecID: "A_OPUS", - TrackType: 2, - DefaultDuration: 20000000, - Audio: &webm.Audio{ - SamplingFrequency: 48000.0, - Channels: 2, - }, - }, { - Name: "Video", - TrackNumber: 2, - TrackUID: 67890, - CodecID: "V_VP8", - TrackType: 1, - DefaultDuration: 33333333, + Name: "Video", + TrackNumber: 1, + TrackUID: 12345, + CodecID: "V_VP8", + TrackType: 1, Video: &webm.Video{ PixelWidth: uint64(width), PixelHeight: uint64(height), }, }, + { + Name: "Audio", + TrackNumber: 2, + TrackUID: 54321, + CodecID: "A_OPUS", + TrackType: 2, + Audio: &webm.Audio{ + SamplingFrequency: 48000.0, + Channels: 2, + }, + }, }, mkvcore.WithSegmentInfo(info)) + if err != nil { panic(err) } log.WithField("session", r.ctx.Value("session")). Printf("webm writer started with %dx%d video: %s", width, height, r.file) - r.audioWriter = ws[0] - r.videoWriter = ws[1] + r.videoWriter = ws[0] + r.audioWriter = ws[1] r.started = true - - r.flowing = true } diff --git a/internal/webrtc/utils/jitter_buffer.go b/internal/webrtc/utils/jitter_buffer.go index a908750..495f8e3 100644 --- a/internal/webrtc/utils/jitter_buffer.go +++ b/internal/webrtc/utils/jitter_buffer.go @@ -11,7 +11,7 @@ type JitterBuffer struct { size int64 } -func NewJitterBuffer(size int) *JitterBuffer { +func NewJitterBuffer(size uint16) *JitterBuffer { return &JitterBuffer{ packets: make([]*rtp.Packet, size), size: int64(size), diff --git a/internal/webrtc/utils/nack.go b/internal/webrtc/utils/nack.go index cf143f5..224d342 100644 --- a/internal/webrtc/utils/nack.go +++ b/internal/webrtc/utils/nack.go @@ -28,7 +28,7 @@ func NackPairs(seqNums []uint16) []rtcp.NackPair { return pairs } -func NackParsToSequenceNumbers(pairs []rtcp.NackPair) []uint16 { +func NackPairsToSequenceNumbers(pairs []rtcp.NackPair) []uint16 { seqs := make([]uint16, 0) for _, pair := range pairs { startSeq := pair.PacketID diff --git a/internal/webrtc/utils/nack_test.go b/internal/webrtc/utils/nack_test.go index 5c36a3d..cb185f9 100644 --- a/internal/webrtc/utils/nack_test.go +++ b/internal/webrtc/utils/nack_test.go @@ -48,7 +48,7 @@ func TestNackParsToSequenceNumbers(t *testing.T) { u16(65580), u16(65581), } - seqNums := NackParsToSequenceNumbers(pairs) + seqNums := NackPairsToSequenceNumbers(pairs) if diff := pretty.Diff(wantSeqNums, seqNums); len(diff) > 0 { t.Errorf("want/got: %v", diff) diff --git a/internal/webrtc/webrtc.go b/internal/webrtc/webrtc.go index 6424415..d63880a 100644 --- a/internal/webrtc/webrtc.go +++ b/internal/webrtc/webrtc.go @@ -25,7 +25,12 @@ func NewWebRTC(ctx context.Context, cfg config.WebRTC) *WebRTC { return &WebRTC{ctx: ctx, cfg: cfg} } -func (w WebRTC) Init(offer webrtc.SessionDescription, r recorder.Recorder, connStateCallbackFn func(state webrtc.ICEConnectionState)) webrtc.SessionDescription { +func (w WebRTC) Init( + offer webrtc.SessionDescription, + r recorder.Recorder, + connStateCallbackFn func(state webrtc.ICEConnectionState), + flowCallbackFn func(isFlowing bool, videoTimestamp time.Duration, closed bool), +) webrtc.SessionDescription { // Prepare the configuration cfg := webrtc.Configuration{ ICEServers: w.cfg.ICEServers, @@ -82,67 +87,152 @@ func (w WebRTC) Init(offer webrtc.SessionDescription, r recorder.Recorder, connS } w.pc = peerConnection - //done := make(chan bool) - trackDone := make([]chan bool, 0) + for _, md := range sdpOffer.MediaDescriptions { + switch md.MediaName.Media { + case "audio": + if _, err = peerConnection.AddTransceiverFromKind(webrtc.RTPCodecTypeAudio); err != nil { + panic(err) + } + case "video": + if _, err = peerConnection.AddTransceiverFromKind(webrtc.RTPCodecTypeVideo); err != nil { + panic(err) + } + } + } + // Set a handler for when a new remote track starts, this handler copies inbound RTP packets, // replaces the SSRC and sends them back peerConnection.OnTrack(func(track *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) { - // Send a PLI on an interval so that the publisher is pushing a keyframe every rtcpPLIInterval - ticker := time.NewTicker(time.Second * 3) - done1 := make(chan bool) - trackDone = append(trackDone, done1) - go func() { - for { - select { - case <-done1: - ticker.Stop() - return - case <-ticker.C: - errSend := peerConnection.WriteRTCP([]rtcp.Packet{ - &rtcp.PictureLossIndication{ - MediaSSRC: uint32(track.SSRC()), - }, - }) - if errSend != nil { - log.WithField("session", w.ctx.Value("session")). - Error(errSend) + isAudio := track.Kind() == webrtc.RTPCodecTypeAudio + isVideo := track.Kind() == webrtc.RTPCodecTypeVideo + + log.WithField("session", w.ctx.Value("session")). + Infof("%s (%d) track started", track.Codec().RTPCodecCapability.MimeType, track.PayloadType()) + + if isVideo { + // Send a PLI on an interval so that the publisher is pushing a keyframe every rtcpPLIInterval + done := make(chan bool) + defer func() { + done <- true + }() + go func() { + ticker := time.NewTicker(time.Second * 3) + for { + select { + case <-done: + ticker.Stop() + return + case <-ticker.C: + errSend := peerConnection.WriteRTCP([]rtcp.Packet{ + &rtcp.PictureLossIndication{ + MediaSSRC: uint32(track.SSRC()), + }, + }) + if errSend != nil { + log.WithField("session", w.ctx.Value("session")). + Error(errSend) + } } } - } - }() + }() + } - receiveLog, err := utils.NewReceiveLog(1024) + rl, err := utils.NewReceiveLog(1024) if err != nil { panic(err) } - var senderSSRC = rand.Uint32() - done2 := make(chan bool) - trackDone = append(trackDone, done2) - go func() { - ticker := time.NewTicker(time.Millisecond * 50) - for { - select { - case <-done2: - ticker.Stop() - return - case <-ticker.C: - missing := receiveLog.MissingSeqNumbers(10) - nack := &rtcp.TransportLayerNack{ - SenderSSRC: senderSSRC, - MediaSSRC: uint32(track.SSRC()), - Nacks: utils.NackPairs(missing), + + if true { + senderSSRC := rand.Uint32() + done := make(chan bool) + defer func() { + done <- true + }() + go func() { + ticker := time.NewTicker(time.Millisecond * 50) + for { + select { + case <-done: + ticker.Stop() + return + case <-ticker.C: + missing := rl.MissingSeqNumbers(10) + if missing == nil || len(missing) == 0 { + continue + } + nack := &rtcp.TransportLayerNack{ + SenderSSRC: senderSSRC, + MediaSSRC: uint32(track.SSRC()), + Nacks: utils.NackPairs(missing), + } + errSend := peerConnection.WriteRTCP([]rtcp.Packet{nack}) + if errSend != nil { + log.WithField("session", w.ctx.Value("session")). + Error(errSend) + } } - errSend := peerConnection.WriteRTCP([]rtcp.Packet{nack}) - if errSend != nil { - log.WithField("session", w.ctx.Value("session")). - Error(errSend) + } + }() + } + + jb := utils.NewJitterBuffer(w.cfg.JitterBuffer) + + if true { + done := make(chan bool) + defer func() { + done <- true + }() + go func() { + ticker := time.NewTicker(time.Millisecond * 100) + var wasFlowing, isFlowing bool + var s1, s2 uint16 + for { + select { + case <-done: + ticker.Stop() + if isVideo { + flowCallbackFn(false, r.VideoTimestamp(), true) + } + return + case <-ticker.C: + if s1 == s2 { + isFlowing = false + } else { + isFlowing = true + } + s1 = s2 + if isVideo { + if isFlowing != wasFlowing { + flowCallbackFn(isFlowing, r.VideoTimestamp(), false) + if isFlowing { + ticker.Reset(time.Millisecond * 1000) + } else { + ticker.Reset(time.Millisecond * 100) + } + } + } + wasFlowing = isFlowing + default: + packets := jb.NextPackets() + if packets == nil { + continue + } + for _, p := range packets { + s2 = p.SequenceNumber + switch { + case isAudio: + r.PushAudio(p) + case isVideo: + r.PushVideo(p) + } + } } } - } - }() + }() + } - log.WithField("session", w.ctx.Value("session")). - Infof("%s (%d) track started", track.Codec().RTPCodecCapability.MimeType, track.PayloadType()) + var seq int64 + su := utils.NewSequenceUnwrapper(16) for { // Read RTP packets being sent to Pion rtp, _, readErr := track.ReadRTP() @@ -156,12 +246,12 @@ func (w WebRTC) Init(offer webrtc.SessionDescription, r recorder.Recorder, connS Error(readErr) return } - - receiveLog.Add(rtp.SequenceNumber) - - r.Push(rtp, track) + seq = su.Unwrap(uint64(rtp.SequenceNumber)) + jb.Add(seq, rtp) + rl.Add(rtp.SequenceNumber) } }) + // Set the handler for ICE connection state // This will notify you when the peer has connected/disconnected peerConnection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) { @@ -172,11 +262,6 @@ func (w WebRTC) Init(offer webrtc.SessionDescription, r recorder.Recorder, connS log.WithField("session", w.ctx.Value("session")). Error(err) } - - for _, done := range trackDone { - done <- true - } - trackDone = nil } connStateCallbackFn(connectionState) }) diff --git a/pkg/ebml-go/.gitignore b/pkg/ebml-go/.gitignore new file mode 100644 index 0000000..c57100a --- /dev/null +++ b/pkg/ebml-go/.gitignore @@ -0,0 +1 @@ +coverage.txt diff --git a/pkg/ebml-go/172-null-term.patch b/pkg/ebml-go/172-null-term.patch new file mode 100644 index 0000000..639d942 --- /dev/null +++ b/pkg/ebml-go/172-null-term.patch @@ -0,0 +1,129 @@ +From 81de5ef1e243a54bbc713a44a294a8a2204e4ada Mon Sep 17 00:00:00 2001 +From: Jorge Villatoro +Date: Thu, 4 Aug 2022 18:41:13 -0700 +Subject: [PATCH] fix: stop adding unnecessary null terminators to strings + +--- + marshal_test.go | 30 +++++++++++++++--------------- + value.go | 4 ++-- + value_test.go | 2 +- + 3 files changed, 18 insertions(+), 18 deletions(-) + +diff --git a/marshal_test.go b/marshal_test.go +index 296dc41..44ecd7f 100644 +--- a/marshal_test.go ++++ b/marshal_test.go +@@ -79,8 +79,8 @@ func TestMarshal(t *testing.T) { + &struct{ EBML TestNoOmitempty }{}, + [][]byte{ + { +- 0x1A, 0x45, 0xDF, 0xA3, 0x8B, +- 0x42, 0x82, 0x81, 0x00, ++ 0x1A, 0x45, 0xDF, 0xA3, 0x8A, ++ 0x42, 0x82, 0x80, + 0x42, 0x87, 0x81, 0x00, + 0x53, 0xAB, 0x80, + }, +@@ -127,8 +127,8 @@ func TestMarshal(t *testing.T) { + &struct{ EBML TestSized }{TestSized{"abc", 0x012345, 0.0, 0.0, []byte{0x01, 0x02, 0x03}}}, + [][]byte{ + { +- 0x1A, 0x45, 0xDF, 0xA3, 0xA5, +- 0x42, 0x82, 0x84, 0x61, 0x62, 0x63, 0x00, ++ 0x1A, 0x45, 0xDF, 0xA3, 0xA4, ++ 0x42, 0x82, 0x83, 0x61, 0x62, 0x63, + 0x42, 0x87, 0x83, 0x01, 0x23, 0x45, + 0x44, 0x89, 0x88, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x44, 0x89, 0x84, 0x00, 0x00, 0x00, 0x00, +@@ -140,8 +140,8 @@ func TestMarshal(t *testing.T) { + &struct{ EBML TestPtr }{TestPtr{&str, &uinteger}}, + [][]byte{ + { +- 0x1A, 0x45, 0xDF, 0xA3, 0x88, +- 0x42, 0x82, 0x81, 0x00, ++ 0x1A, 0x45, 0xDF, 0xA3, 0x87, ++ 0x42, 0x82, 0x80, + 0x42, 0x87, 0x81, 0x00, + }, + }, +@@ -154,8 +154,8 @@ func TestMarshal(t *testing.T) { + &struct{ EBML TestInterface }{TestInterface{str, uinteger}}, + [][]byte{ + { +- 0x1A, 0x45, 0xDF, 0xA3, 0x88, +- 0x42, 0x82, 0x81, 0x00, ++ 0x1A, 0x45, 0xDF, 0xA3, 0x87, ++ 0x42, 0x82, 0x80, + 0x42, 0x87, 0x81, 0x00, + }, + }, +@@ -164,8 +164,8 @@ func TestMarshal(t *testing.T) { + &struct{ EBML TestInterface }{TestInterface{&str, &uinteger}}, + [][]byte{ + { +- 0x1A, 0x45, 0xDF, 0xA3, 0x88, +- 0x42, 0x82, 0x81, 0x00, ++ 0x1A, 0x45, 0xDF, 0xA3, 0x87, ++ 0x42, 0x82, 0x80, + 0x42, 0x87, 0x81, 0x00, + }, + }, +@@ -179,9 +179,9 @@ func TestMarshal(t *testing.T) { + }, + [][]byte{ + { +- 0x15, 0x49, 0xA9, 0x66, 0x90, +- 0x4D, 0x80, 0x85, 0x74, 0x65, 0x73, 0x74, 0x00, +- 0x57, 0x41, 0x85, 0x61, 0x62, 0x63, 0x64, 0x00, ++ 0x15, 0x49, 0xA9, 0x66, 0x8E, ++ 0x4D, 0x80, 0x84, 0x74, 0x65, 0x73, 0x74, ++ 0x57, 0x41, 0x84, 0x61, 0x62, 0x63, 0x64, + }, + { // Go map element order is unstable + 0x15, 0x49, 0xA9, 0x66, 0x90, +@@ -392,7 +392,7 @@ func ExampleMarshal() { + fmt.Printf("0x%02x, ", int(b)) + } + // Output: +- // 0x1a, 0x45, 0xdf, 0xa3, 0x90, 0x42, 0x82, 0x85, 0x77, 0x65, 0x62, 0x6d, 0x00, 0x42, 0x87, 0x81, 0x02, 0x42, 0x85, 0x81, 0x02, ++ // 0x1a, 0x45, 0xdf, 0xa3, 0x8f, 0x42, 0x82, 0x84, 0x77, 0x65, 0x62, 0x6d, 0x42, 0x87, 0x81, 0x02, 0x42, 0x85, 0x81, 0x02, + } + + func ExampleWithDataSizeLen() { +@@ -420,7 +420,7 @@ func ExampleWithDataSizeLen() { + fmt.Printf("0x%02x, ", int(b)) + } + // Output: +- // 0x1a, 0x45, 0xdf, 0xa3, 0x40, 0x13, 0x42, 0x82, 0x40, 0x05, 0x77, 0x65, 0x62, 0x6d, 0x00, 0x42, 0x87, 0x40, 0x01, 0x02, 0x42, 0x85, 0x40, 0x01, 0x02, ++ // 0x1a, 0x45, 0xdf, 0xa3, 0x40, 0x12, 0x42, 0x82, 0x40, 0x04, 0x77, 0x65, 0x62, 0x6d, 0x42, 0x87, 0x40, 0x01, 0x02, 0x42, 0x85, 0x40, 0x01, 0x02, + } + + func TestMarshal_Tag(t *testing.T) { +diff --git a/value.go b/value.go +index fe5771a..81e63c2 100644 +--- a/value.go ++++ b/value.go +@@ -358,8 +358,8 @@ func encodeString(i interface{}, n uint64) ([]byte, error) { + if !ok { + return []byte{}, wrapErrorf(ErrInvalidType, "writing %T as string", i) + } +- if uint64(len(v)+1) >= n { +- return append([]byte(v), 0x00), nil ++ if uint64(len(v)) >= n { ++ return append([]byte(v)), nil + } + return append([]byte(v), bytes.Repeat([]byte{0x00}, int(n)-len(v))...), nil + } +diff --git a/value_test.go b/value_test.go +index 13bd870..89b8a9b 100644 +--- a/value_test.go ++++ b/value_test.go +@@ -204,7 +204,7 @@ func TestValue(t *testing.T) { + }{ + "Binary": {[]byte{0x01, 0x02, 0x03}, DataTypeBinary, []byte{0x01, 0x02, 0x03}, 0, nil}, + "Binary(4B)": {[]byte{0x01, 0x02, 0x03, 0x00}, DataTypeBinary, []byte{0x01, 0x02, 0x03, 0x00}, 4, []byte{0x01, 0x02, 0x03}}, +- "String": {[]byte{0x31, 0x32, 0x00}, DataTypeString, "12", 0, nil}, ++ "String": {[]byte{0x31, 0x32}, DataTypeString, "12", 0, nil}, + "String(3B)": {[]byte{0x31, 0x32, 0x00}, DataTypeString, "12", 3, nil}, + "String(4B)": {[]byte{0x31, 0x32, 0x00, 0x00}, DataTypeString, "12", 4, nil}, + "Int8": {[]byte{0x01}, DataTypeInt, int64(0x01), 0, nil}, diff --git a/pkg/ebml-go/LICENSE b/pkg/ebml-go/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/pkg/ebml-go/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/pkg/ebml-go/README.md b/pkg/ebml-go/README.md new file mode 100644 index 0000000..a6e2996 --- /dev/null +++ b/pkg/ebml-go/README.md @@ -0,0 +1,30 @@ +# ebml-go + +[![GoDoc](https://godoc.org/github.com/at-wat/ebml-go?status.svg)](http://godoc.org/github.com/at-wat/ebml-go) ![ci](https://github.com/at-wat/ebml-go/workflows/ci/badge.svg) [![codecov](https://codecov.io/gh/at-wat/ebml-go/branch/master/graph/badge.svg)](https://codecov.io/gh/at-wat/ebml-go) [![Go Report Card](https://goreportcard.com/badge/github.com/at-wat/ebml-go)](https://goreportcard.com/report/github.com/at-wat/ebml-go) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) + +## A pure Go implementation of bi-directional EBML encoder/decoder + +EBML (Extensible Binary Meta Language) is a binary and byte-aligned format that was originally developed for the Matroska audio-visual container. +See https://matroska.org/ for details. + +This package implements EBML Marshaler and Unmarshaler for Go. +Currently, commonly used elements of WebM subset is supported. + + +## Usage + +Check out the examples placed under [./examples](./examples/) directory. + +API is documented using [GoDoc](http://godoc.org/github.com/at-wat/ebml-go). +EBML can be `Marshal`-ed and `Unmarshal`-ed between tagged struct and binary stream through `io.Reader` and `io.Writer`. + + +## References + +- [Matroska Container Specifications](https://matroska.org/technical/specs/index.html) +- [WebM Container Guidelines](https://www.webmproject.org/docs/container/) + + +## License + +This package is licensed under [Apache License Version 2.0](./LICENSE). diff --git a/pkg/ebml-go/block.go b/pkg/ebml-go/block.go new file mode 100644 index 0000000..a213422 --- /dev/null +++ b/pkg/ebml-go/block.go @@ -0,0 +1,168 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ebml + +import ( + "io" +) + +// LacingMode is type of laced data. +type LacingMode uint8 + +// Type of laced data. +const ( + LacingNo LacingMode = 0 + LacingXiph LacingMode = 1 + LacingFixed LacingMode = 2 + LacingEBML LacingMode = 3 +) + +const ( + blockFlagMaskKeyframe = 0x80 + blockFlagMaskInvisible = 0x08 + blockFlagMaskLacing = 0x06 + blockFlagMaskDiscardable = 0x01 +) + +// Block represents EBML Block/SimpleBlock element. +type Block struct { + TrackNumber uint64 + Timecode int16 + Keyframe bool + Invisible bool + Lacing LacingMode + Discardable bool + Data [][]byte +} + +func (b *Block) packFlags() byte { + var f byte + if b.Keyframe { + f |= blockFlagMaskKeyframe + } + if b.Invisible { + f |= blockFlagMaskInvisible + } + if b.Discardable { + f |= blockFlagMaskDiscardable + } + f |= byte(b.Lacing) << 1 + return f +} + +// UnmarshalBlock unmarshals EBML Block structure. +func UnmarshalBlock(r io.Reader, n int64) (*Block, error) { + var b Block + var err error + var nRead int + + vd := &valueDecoder{} + + if b.TrackNumber, nRead, err = vd.readVUInt(r); err != nil { + return nil, err + } + n -= int64(nRead) + if v, err := vd.readInt(r, 2); err == nil { + b.Timecode = int16(v.(int64)) + } else { + return nil, err + } + n -= 2 + + switch _, err := r.Read(vd.bs[:]); err { + case nil: + case io.EOF: + return nil, io.ErrUnexpectedEOF + default: + return nil, err + } + n-- + + if n < 0 { + return nil, io.ErrUnexpectedEOF + } + + f := vd.bs[0] + if f&blockFlagMaskKeyframe != 0 { + b.Keyframe = true + } + if f&blockFlagMaskInvisible != 0 { + b.Invisible = true + } + if f&blockFlagMaskDiscardable != 0 { + b.Discardable = true + } + b.Lacing = LacingMode((f & blockFlagMaskLacing) >> 1) + + var ul Unlacer + switch b.Lacing { + case LacingNo: + ul, err = NewNoUnlacer(r, n) + case LacingXiph: + ul, err = NewXiphUnlacer(r, n) + case LacingEBML: + ul, err = NewEBMLUnlacer(r, n) + case LacingFixed: + ul, err = NewFixedUnlacer(r, n) + } + if err != nil { + return nil, err + } + + for { + frame, err := ul.Read() + if err == io.EOF { + return &b, nil + } + if err != nil { + return nil, err + } + b.Data = append(b.Data, frame) + } +} + +// MarshalBlock marshals EBML Block structure. +func MarshalBlock(b *Block, w io.Writer) error { + n, err := encodeElementID(b.TrackNumber) + if err != nil { + return err + } + if _, err := w.Write(n); err != nil { + return err + } + if _, err := w.Write([]byte{byte(b.Timecode >> 8), byte(b.Timecode)}); err != nil { + return err + } + if _, err := w.Write([]byte{b.packFlags()}); err != nil { + return err + } + + var l Lacer + switch b.Lacing { + case LacingNo: + l = NewNoLacer(w) + case LacingXiph: + l = NewXiphLacer(w) + case LacingEBML: + l = NewEBMLLacer(w) + case LacingFixed: + l = NewFixedLacer(w) + } + if err := l.Write(b.Data); err != nil { + return err + } + + return nil +} diff --git a/pkg/ebml-go/block_test.go b/pkg/ebml-go/block_test.go new file mode 100644 index 0000000..ddfafa4 --- /dev/null +++ b/pkg/ebml-go/block_test.go @@ -0,0 +1,163 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ebml + +import ( + "bytes" + "io" + "reflect" + "testing" + + "github.com/at-wat/ebml-go/internal/errs" +) + +func TestUnmarshalBlock(t *testing.T) { + testCases := map[string]struct { + input []byte + expected Block + }{ + "Track1BKeyframeInvisible": { + []byte{0x82, 0x01, 0x23, 0x88, 0xAA, 0xCC}, + Block{0x02, 0x0123, true, true, LacingNo, false, [][]byte{{0xAA, 0xCC}}}, + }, + "Track2BDiscardable": { + []byte{0x42, 0x13, 0x01, 0x23, 0x01, 0x11, 0x22, 0x33}, + Block{0x0213, 0x0123, false, false, LacingNo, true, [][]byte{{0x11, 0x22, 0x33}}}, + }, + "Track3BNoData": { + []byte{0x21, 0x23, 0x45, 0x00, 0x02, 0x00}, + Block{0x012345, 0x0002, false, false, LacingNo, false, [][]byte{{}}}, + }, + "FixedLace": { + []byte{0x82, 0x01, 0x23, 0x04, 0x02, 0x0A, 0x0B, 0x0C}, + Block{ + 0x02, 0x0123, false, false, LacingFixed, false, + [][]byte{{0x0A}, {0x0B}, {0x0C}}, + }, + }, + "XiphLace": { + []byte{0x82, 0x01, 0x23, 0x02, 0x02, 0x01, 0x02, 0x0A, 0x0B, 0x1B, 0x0C}, + Block{ + 0x02, 0x0123, false, false, LacingXiph, false, + [][]byte{{0x0A}, {0x0B, 0x1B}, {0x0C}}, + }, + }, + "EBMLLace": { + []byte{0x82, 0x01, 0x23, 0x06, 0x02, 0x81, 0xC0, 0x0A, 0x0B, 0x1B, 0x0C}, + Block{ + 0x02, 0x0123, false, false, LacingEBML, false, + [][]byte{{0x0A}, {0x0B, 0x1B}, {0x0C}}, + }, + }, + } + for n, c := range testCases { + t.Run(n, func(t *testing.T) { + block, err := UnmarshalBlock(bytes.NewBuffer(c.input), int64(len(c.input))) + if err != nil { + t.Fatalf("Failed to unmarshal block: '%v'", err) + } + if !reflect.DeepEqual(c.expected, *block) { + t.Errorf("Expected unmarshal result: '%v', got: '%v'", c.expected, *block) + } + }) + } +} + +func TestUnmarshalBlock_Error(t *testing.T) { + t.Run("EOF", func(t *testing.T) { + input := []byte{0x21, 0x23, 0x45, 0x00, 0x02, 0x00} + for l := 0; l < len(input); l++ { + if _, err := UnmarshalBlock(bytes.NewBuffer(input[:l]), int64(len(input))); !errs.Is(err, io.ErrUnexpectedEOF) { + t.Errorf("Short data (%d bytes) expected error: '%v', got: '%v'", + l, io.ErrUnexpectedEOF, err) + } + } + }) + testCases := map[string]struct { + input []byte + err error + }{ + "UndivisibleFixedLace": { + []byte{0x82, 0x00, 0x00, 0x04, 0x02, 0x00, 0x00}, + ErrFixedLaceUndivisible, + }, + } + for n, c := range testCases { + t.Run(n, func(t *testing.T) { + if _, err := UnmarshalBlock(bytes.NewBuffer(c.input), int64(len(c.input))); !errs.Is(err, c.err) { + t.Errorf("Expected error: '%v', got: '%v'", c.err, err) + } + }) + } +} + +func TestMarshalBlock(t *testing.T) { + testCases := map[string]struct { + input Block + expected []byte + }{ + "Track1BKeyframeInvisible": { + Block{0x02, 0x0123, true, true, LacingNo, false, [][]byte{{0xAA, 0xCC}}}, + []byte{0x82, 0x01, 0x23, 0x88, 0xAA, 0xCC}, + }, + "Track2BDiscardable": { + Block{0x0213, 0x0123, false, false, LacingNo, true, [][]byte{{0x11, 0x22, 0x33}}}, + []byte{0x42, 0x13, 0x01, 0x23, 0x01, 0x11, 0x22, 0x33}, + }, + "Track3BNoData": { + Block{0x012345, 0x0002, false, false, LacingNo, false, [][]byte{{}}}, + []byte{0x21, 0x23, 0x45, 0x00, 0x02, 0x00}, + }, + } + for n, c := range testCases { + t.Run(n, func(t *testing.T) { + var b bytes.Buffer + if err := MarshalBlock(&c.input, &b); err != nil { + t.Fatalf("Failed to marshal block: '%v'", err) + } + if !reflect.DeepEqual(c.expected, b.Bytes()) { + t.Errorf("Expected marshal result: '%v', got: '%v'", c.expected, b.Bytes()) + } + }) + } +} + +func TestMarshalBlock_Error(t *testing.T) { + cases := map[string]struct { + input *Block + err error + }{ + "InvalidTrackNum": { + &Block{0xFFFFFFFFFFFFFFFF, 0x0000, false, false, LacingNo, false, [][]byte{{}}}, + ErrUnsupportedElementID, + }, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + if err := MarshalBlock(c.input, &bytes.Buffer{}); !errs.Is(err, c.err) { + t.Errorf("Expected error: '%v', got: '%v'", c.err, err) + } + }) + } + + t.Run("EOF", func(t *testing.T) { + input := &Block{0x012345, 0x0002, false, false, LacingNo, false, [][]byte{{0x00}}} // 7 bytes + for l := 0; l < 7; l++ { + if err := MarshalBlock(input, &limitedDummyWriter{limit: l}); !errs.Is(err, bytes.ErrTooLarge) { + t.Errorf("Expected error against too large data (Writer size limit: %d): '%v', got: '%v'", l, bytes.ErrTooLarge, err) + } + } + }) +} diff --git a/pkg/ebml-go/codecov.yml b/pkg/ebml-go/codecov.yml new file mode 100644 index 0000000..ec00c1d --- /dev/null +++ b/pkg/ebml-go/codecov.yml @@ -0,0 +1,10 @@ +coverage: + status: + project: + default: + target: auto + threshold: 2% + patch: + default: + target: 75% + base: pr diff --git a/pkg/ebml-go/datatype.go b/pkg/ebml-go/datatype.go new file mode 100644 index 0000000..dad043b --- /dev/null +++ b/pkg/ebml-go/datatype.go @@ -0,0 +1,79 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ebml + +import ( + "reflect" +) + +// DataType represents EBML Element data type. +type DataType int + +// EBML Element data types. +const ( + DataTypeMaster DataType = iota + DataTypeInt + DataTypeUInt + DataTypeDate + DataTypeFloat + DataTypeBinary + DataTypeString + DataTypeBlock +) + +var dataTypeName = map[DataType]string{ + DataTypeMaster: "Master", + DataTypeInt: "Int", + DataTypeUInt: "UInt", + DataTypeDate: "Date", + DataTypeFloat: "Float", + DataTypeBinary: "Binary", + DataTypeString: "String", + DataTypeBlock: "Block", +} + +func (t DataType) String() string { + if name, ok := dataTypeName[t]; ok { + return name + } + return "unknown" +} + +func isConvertible(src, dst reflect.Type) bool { + switch src.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + switch dst.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return true + default: + return false + } + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + switch dst.Kind() { + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return true + default: + return false + } + case reflect.Float32, reflect.Float64: + switch dst.Kind() { + case reflect.Float32, reflect.Float64: + return true + default: + return false + } + } + return false +} diff --git a/pkg/ebml-go/datatype_test.go b/pkg/ebml-go/datatype_test.go new file mode 100644 index 0000000..01e7511 --- /dev/null +++ b/pkg/ebml-go/datatype_test.go @@ -0,0 +1,26 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ebml + +import ( + "testing" +) + +func TestDataType_String(t *testing.T) { + invalid := DataType(-1) + if invalid.String() != "unknown" { + t.Errorf("Invalid DataType string should be 'unknown', got '%s'", invalid.String()) + } +} diff --git a/pkg/ebml-go/ebml.go b/pkg/ebml-go/ebml.go new file mode 100644 index 0000000..aa9d159 --- /dev/null +++ b/pkg/ebml-go/ebml.go @@ -0,0 +1,19 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package ebml implements the encoder and decoder of Extensible Binary Meta Language (EBML). +// +// The package supports Marshal and Unmarshal between tagged struct and EBML binary stream. +// WebM block data writer is provided as webm sub-package. +package ebml diff --git a/pkg/ebml-go/elementtable.go b/pkg/ebml-go/elementtable.go new file mode 100644 index 0000000..34e0aa3 --- /dev/null +++ b/pkg/ebml-go/elementtable.go @@ -0,0 +1,283 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ebml + +import ( + "bytes" +) + +type elementDef struct { + b []byte + t DataType + top bool +} +type elementTable map[ElementType]elementDef + +var table = elementTable{ + ElementChapters: elementDef{[]byte{0x10, 0x43, 0xA7, 0x70}, DataTypeMaster, true}, + ElementSeekHead: elementDef{[]byte{0x11, 0x4D, 0x9B, 0x74}, DataTypeMaster, true}, + ElementTags: elementDef{[]byte{0x12, 0x54, 0xC3, 0x67}, DataTypeMaster, true}, + ElementInfo: elementDef{[]byte{0x15, 0x49, 0xA9, 0x66}, DataTypeMaster, true}, + ElementTracks: elementDef{[]byte{0x16, 0x54, 0xAE, 0x6B}, DataTypeMaster, true}, + ElementSegment: elementDef{[]byte{0x18, 0x53, 0x80, 0x67}, DataTypeMaster, false}, + ElementAttachments: elementDef{[]byte{0x19, 0x41, 0xA4, 0x69}, DataTypeMaster, true}, + ElementEBML: elementDef{[]byte{0x1A, 0x45, 0xDF, 0xA3}, DataTypeMaster, false}, + ElementCues: elementDef{[]byte{0x1C, 0x53, 0xBB, 0x6B}, DataTypeMaster, true}, + ElementCluster: elementDef{[]byte{0x1F, 0x43, 0xB6, 0x75}, DataTypeMaster, true}, + ElementLanguage: elementDef{[]byte{0x22, 0xB5, 0x9C}, DataTypeString, false}, + ElementLanguageIETF: elementDef{[]byte{0x22, 0xB5, 0x9D}, DataTypeString, false}, + ElementTrackTimestampScale: elementDef{[]byte{0x23, 0x31, 0x4F}, DataTypeFloat, false}, + ElementDefaultDecodedFieldDuration: elementDef{[]byte{0x23, 0x4E, 0x7A}, DataTypeUInt, false}, + ElementDefaultDuration: elementDef{[]byte{0x23, 0xE3, 0x83}, DataTypeUInt, false}, + ElementCodecName: elementDef{[]byte{0x25, 0x86, 0x88}, DataTypeString, false}, + ElementTimecodeScale: elementDef{[]byte{0x2A, 0xD7, 0xB1}, DataTypeUInt, false}, + ElementColourSpace: elementDef{[]byte{0x2E, 0xB5, 0x24}, DataTypeBinary, false}, + ElementPrevFilename: elementDef{[]byte{0x3C, 0x83, 0xAB}, DataTypeString, false}, + ElementPrevUID: elementDef{[]byte{0x3C, 0xB9, 0x23}, DataTypeBinary, false}, + ElementNextFilename: elementDef{[]byte{0x3E, 0x83, 0xBB}, DataTypeString, false}, + ElementNextUID: elementDef{[]byte{0x3E, 0xB9, 0x23}, DataTypeBinary, false}, + ElementBlockAddIDName: elementDef{[]byte{0x41, 0xA4}, DataTypeString, false}, + ElementBlockAdditionMapping: elementDef{[]byte{0x41, 0xE4}, DataTypeMaster, false}, + ElementBlockAddIDType: elementDef{[]byte{0x41, 0xE7}, DataTypeUInt, false}, + ElementBlockAddIDExtraData: elementDef{[]byte{0x41, 0xED}, DataTypeBinary, false}, + ElementBlockAddIDValue: elementDef{[]byte{0x41, 0xF0}, DataTypeUInt, false}, + ElementContentCompAlgo: elementDef{[]byte{0x42, 0x54}, DataTypeUInt, false}, + ElementContentCompSettings: elementDef{[]byte{0x42, 0x55}, DataTypeBinary, false}, + ElementEBMLVersion: elementDef{[]byte{0x42, 0x86}, DataTypeUInt, false}, + ElementEBMLMaxIDLength: elementDef{[]byte{0x42, 0xF2}, DataTypeUInt, false}, + ElementEBMLMaxSizeLength: elementDef{[]byte{0x42, 0xF3}, DataTypeUInt, false}, + ElementEBMLReadVersion: elementDef{[]byte{0x42, 0xF7}, DataTypeUInt, false}, + ElementEBMLDocType: elementDef{[]byte{0x42, 0x82}, DataTypeString, false}, + ElementEBMLDocTypeReadVersion: elementDef{[]byte{0x42, 0x85}, DataTypeUInt, false}, + ElementEBMLDocTypeVersion: elementDef{[]byte{0x42, 0x87}, DataTypeUInt, false}, + ElementChapLanguage: elementDef{[]byte{0x43, 0x7C}, DataTypeString, false}, + ElementChapLanguageIETF: elementDef{[]byte{0x43, 0x7D}, DataTypeString, false}, + ElementChapCountry: elementDef{[]byte{0x43, 0x7E}, DataTypeString, false}, + ElementSegmentFamily: elementDef{[]byte{0x44, 0x44}, DataTypeBinary, false}, + ElementDateUTC: elementDef{[]byte{0x44, 0x61}, DataTypeDate, false}, + ElementTagLanguage: elementDef{[]byte{0x44, 0x7A}, DataTypeString, false}, + ElementTagLanguageIETF: elementDef{[]byte{0x44, 0x7B}, DataTypeString, false}, + ElementTagDefault: elementDef{[]byte{0x44, 0x84}, DataTypeUInt, false}, + ElementTagBinary: elementDef{[]byte{0x44, 0x85}, DataTypeBinary, false}, + ElementTagString: elementDef{[]byte{0x44, 0x87}, DataTypeString, false}, + ElementDuration: elementDef{[]byte{0x44, 0x89}, DataTypeFloat, false}, + ElementChapProcessPrivate: elementDef{[]byte{0x45, 0x0D}, DataTypeBinary, false}, + ElementChapterFlagEnabled: elementDef{[]byte{0x45, 0x98}, DataTypeUInt, false}, + ElementTagName: elementDef{[]byte{0x45, 0xA3}, DataTypeString, false}, + ElementEditionEntry: elementDef{[]byte{0x45, 0xB9}, DataTypeMaster, false}, + ElementEditionUID: elementDef{[]byte{0x45, 0xBC}, DataTypeUInt, false}, + ElementEditionFlagHidden: elementDef{[]byte{0x45, 0xBD}, DataTypeUInt, false}, + ElementEditionFlagDefault: elementDef{[]byte{0x45, 0xDB}, DataTypeUInt, false}, + ElementEditionFlagOrdered: elementDef{[]byte{0x45, 0xDD}, DataTypeUInt, false}, + ElementFileData: elementDef{[]byte{0x46, 0x5C}, DataTypeBinary, false}, + ElementFileMimeType: elementDef{[]byte{0x46, 0x60}, DataTypeString, false}, + ElementFileName: elementDef{[]byte{0x46, 0x6E}, DataTypeString, false}, + ElementFileDescription: elementDef{[]byte{0x46, 0x7E}, DataTypeString, false}, + ElementFileUID: elementDef{[]byte{0x46, 0xAE}, DataTypeUInt, false}, + ElementContentEncAlgo: elementDef{[]byte{0x47, 0xE1}, DataTypeUInt, false}, + ElementContentEncKeyID: elementDef{[]byte{0x47, 0xE2}, DataTypeBinary, false}, + ElementContentSignature: elementDef{[]byte{0x47, 0xE3}, DataTypeBinary, false}, + ElementContentSigKeyID: elementDef{[]byte{0x47, 0xE4}, DataTypeBinary, false}, + ElementContentSigAlgo: elementDef{[]byte{0x47, 0xE5}, DataTypeUInt, false}, + ElementContentSigHashAlgo: elementDef{[]byte{0x47, 0xE6}, DataTypeUInt, false}, + ElementContentEncAESSettings: elementDef{[]byte{0x47, 0xE7}, DataTypeMaster, false}, + ElementAESSettingsCipherMode: elementDef{[]byte{0x47, 0xE8}, DataTypeUInt, false}, + ElementMuxingApp: elementDef{[]byte{0x4D, 0x80}, DataTypeString, false}, + ElementSeek: elementDef{[]byte{0x4D, 0xBB}, DataTypeMaster, false}, + ElementContentEncodingOrder: elementDef{[]byte{0x50, 0x31}, DataTypeUInt, false}, + ElementContentEncodingScope: elementDef{[]byte{0x50, 0x32}, DataTypeUInt, false}, + ElementContentEncodingType: elementDef{[]byte{0x50, 0x33}, DataTypeUInt, false}, + ElementContentCompression: elementDef{[]byte{0x50, 0x34}, DataTypeMaster, false}, + ElementContentEncryption: elementDef{[]byte{0x50, 0x35}, DataTypeMaster, false}, + ElementSeekID: elementDef{[]byte{0x53, 0xAB}, DataTypeBinary, false}, + ElementSeekPosition: elementDef{[]byte{0x53, 0xAC}, DataTypeUInt, false}, + ElementStereoMode: elementDef{[]byte{0x53, 0xB8}, DataTypeUInt, false}, + ElementAlphaMode: elementDef{[]byte{0x53, 0xC0}, DataTypeUInt, false}, + ElementName: elementDef{[]byte{0x53, 0x6E}, DataTypeString, false}, + ElementCueBlockNumber: elementDef{[]byte{0x53, 0x78}, DataTypeUInt, false}, + ElementPixelCropBottom: elementDef{[]byte{0x54, 0xAA}, DataTypeUInt, false}, + ElementDisplayWidth: elementDef{[]byte{0x54, 0xB0}, DataTypeUInt, false}, + ElementDisplayUnit: elementDef{[]byte{0x54, 0xB2}, DataTypeUInt, false}, + ElementAspectRatioType: elementDef{[]byte{0x54, 0xB3}, DataTypeUInt, false}, + ElementDisplayHeight: elementDef{[]byte{0x54, 0xBA}, DataTypeUInt, false}, + ElementPixelCropTop: elementDef{[]byte{0x54, 0xBB}, DataTypeUInt, false}, + ElementPixelCropLeft: elementDef{[]byte{0x54, 0xCC}, DataTypeUInt, false}, + ElementPixelCropRight: elementDef{[]byte{0x54, 0xDD}, DataTypeUInt, false}, + ElementFlagForced: elementDef{[]byte{0x55, 0xAA}, DataTypeUInt, false}, + ElementColour: elementDef{[]byte{0x55, 0xB0}, DataTypeMaster, false}, + ElementMatrixCoefficients: elementDef{[]byte{0x55, 0xB1}, DataTypeUInt, false}, + ElementBitsPerChannel: elementDef{[]byte{0x55, 0xB2}, DataTypeUInt, false}, + ElementChromaSubsamplingHorz: elementDef{[]byte{0x55, 0xB3}, DataTypeUInt, false}, + ElementChromaSubsamplingVert: elementDef{[]byte{0x55, 0xB4}, DataTypeUInt, false}, + ElementCbSubsamplingHorz: elementDef{[]byte{0x55, 0xB5}, DataTypeUInt, false}, + ElementCbSubsamplingVert: elementDef{[]byte{0x55, 0xB6}, DataTypeUInt, false}, + ElementChromaSitingHorz: elementDef{[]byte{0x55, 0xB7}, DataTypeUInt, false}, + ElementChromaSitingVert: elementDef{[]byte{0x55, 0xB8}, DataTypeUInt, false}, + ElementRange: elementDef{[]byte{0x55, 0xB9}, DataTypeUInt, false}, + ElementTransferCharacteristics: elementDef{[]byte{0x55, 0xBA}, DataTypeUInt, false}, + ElementPrimaries: elementDef{[]byte{0x55, 0xBB}, DataTypeUInt, false}, + ElementMaxCLL: elementDef{[]byte{0x55, 0xBC}, DataTypeUInt, false}, + ElementMaxFALL: elementDef{[]byte{0x55, 0xBD}, DataTypeUInt, false}, + ElementMasteringMetadata: elementDef{[]byte{0x55, 0xD0}, DataTypeMaster, false}, + ElementPrimaryRChromaticityX: elementDef{[]byte{0x55, 0xD1}, DataTypeFloat, false}, + ElementPrimaryRChromaticityY: elementDef{[]byte{0x55, 0xD2}, DataTypeFloat, false}, + ElementPrimaryGChromaticityX: elementDef{[]byte{0x55, 0xD3}, DataTypeFloat, false}, + ElementPrimaryGChromaticityY: elementDef{[]byte{0x55, 0xD4}, DataTypeFloat, false}, + ElementPrimaryBChromaticityX: elementDef{[]byte{0x55, 0xD5}, DataTypeFloat, false}, + ElementPrimaryBChromaticityY: elementDef{[]byte{0x55, 0xD6}, DataTypeFloat, false}, + ElementWhitePointChromaticityX: elementDef{[]byte{0x55, 0xD7}, DataTypeFloat, false}, + ElementWhitePointChromaticityY: elementDef{[]byte{0x55, 0xD8}, DataTypeFloat, false}, + ElementLuminanceMax: elementDef{[]byte{0x55, 0xD9}, DataTypeFloat, false}, + ElementLuminanceMin: elementDef{[]byte{0x55, 0xDA}, DataTypeFloat, false}, + ElementMaxBlockAdditionID: elementDef{[]byte{0x55, 0xEE}, DataTypeUInt, false}, + ElementChapterStringUID: elementDef{[]byte{0x56, 0x54}, DataTypeString, false}, + ElementCodecDelay: elementDef{[]byte{0x56, 0xAA}, DataTypeUInt, false}, + ElementSeekPreRoll: elementDef{[]byte{0x56, 0xBB}, DataTypeUInt, false}, + ElementWritingApp: elementDef{[]byte{0x57, 0x41}, DataTypeString, false}, + ElementSilentTracks: elementDef{[]byte{0x58, 0x54}, DataTypeMaster, false}, + ElementSilentTrackNumber: elementDef{[]byte{0x58, 0xD7}, DataTypeUInt, false}, + ElementAttachedFile: elementDef{[]byte{0x61, 0xA7}, DataTypeMaster, false}, + ElementContentEncoding: elementDef{[]byte{0x62, 0x40}, DataTypeMaster, false}, + ElementBitDepth: elementDef{[]byte{0x62, 0x64}, DataTypeUInt, false}, + ElementCodecPrivate: elementDef{[]byte{0x63, 0xA2}, DataTypeBinary, false}, + ElementTargets: elementDef{[]byte{0x63, 0xC0}, DataTypeMaster, false}, + ElementChapterPhysicalEquiv: elementDef{[]byte{0x63, 0xC3}, DataTypeUInt, false}, + ElementTagChapterUID: elementDef{[]byte{0x63, 0xC4}, DataTypeUInt, false}, + ElementTagTrackUID: elementDef{[]byte{0x63, 0xC5}, DataTypeUInt, false}, + ElementTagAttachmentUID: elementDef{[]byte{0x63, 0xC6}, DataTypeUInt, false}, + ElementTagEditionUID: elementDef{[]byte{0x63, 0xC9}, DataTypeUInt, false}, + ElementTargetType: elementDef{[]byte{0x63, 0xCA}, DataTypeString, false}, + ElementTrackTranslate: elementDef{[]byte{0x66, 0x24}, DataTypeMaster, false}, + ElementTrackTranslateTrackID: elementDef{[]byte{0x66, 0xA5}, DataTypeBinary, false}, + ElementTrackTranslateCodec: elementDef{[]byte{0x66, 0xBF}, DataTypeUInt, false}, + ElementTrackTranslateEditionUID: elementDef{[]byte{0x66, 0xFC}, DataTypeUInt, false}, + ElementSimpleTag: elementDef{[]byte{0x67, 0xC8}, DataTypeMaster, false}, + ElementTargetTypeValue: elementDef{[]byte{0x68, 0xCA}, DataTypeUInt, false}, + ElementChapProcessCommand: elementDef{[]byte{0x69, 0x11}, DataTypeMaster, false}, + ElementChapProcessTime: elementDef{[]byte{0x69, 0x22}, DataTypeUInt, false}, + ElementChapterTranslate: elementDef{[]byte{0x69, 0x24}, DataTypeMaster, false}, + ElementChapProcessData: elementDef{[]byte{0x69, 0x33}, DataTypeBinary, false}, + ElementChapProcess: elementDef{[]byte{0x69, 0x44}, DataTypeMaster, false}, + ElementChapProcessCodecID: elementDef{[]byte{0x69, 0x55}, DataTypeUInt, false}, + ElementChapterTranslateID: elementDef{[]byte{0x69, 0xA5}, DataTypeBinary, false}, + ElementChapterTranslateCodec: elementDef{[]byte{0x69, 0xBF}, DataTypeUInt, false}, + ElementChapterTranslateEditionUID: elementDef{[]byte{0x69, 0xFC}, DataTypeUInt, false}, + ElementContentEncodings: elementDef{[]byte{0x6D, 0x80}, DataTypeMaster, false}, + ElementMinCache: elementDef{[]byte{0x6D, 0xE7}, DataTypeUInt, false}, + ElementMaxCache: elementDef{[]byte{0x6D, 0xF8}, DataTypeUInt, false}, + ElementChapterSegmentUID: elementDef{[]byte{0x6E, 0x67}, DataTypeBinary, false}, + ElementChapterSegmentEditionUID: elementDef{[]byte{0x6E, 0xBC}, DataTypeUInt, false}, + ElementTrackOverlay: elementDef{[]byte{0x6F, 0xAB}, DataTypeUInt, false}, + ElementTag: elementDef{[]byte{0x73, 0x73}, DataTypeMaster, false}, + ElementSegmentFilename: elementDef{[]byte{0x73, 0x84}, DataTypeString, false}, + ElementSegmentUID: elementDef{[]byte{0x73, 0xA4}, DataTypeBinary, false}, + ElementChapterUID: elementDef{[]byte{0x73, 0xC4}, DataTypeUInt, false}, + ElementTrackUID: elementDef{[]byte{0x73, 0xC5}, DataTypeUInt, false}, + ElementAttachmentLink: elementDef{[]byte{0x74, 0x46}, DataTypeUInt, false}, + ElementBlockAdditions: elementDef{[]byte{0x75, 0xA1}, DataTypeMaster, false}, + ElementDiscardPadding: elementDef{[]byte{0x75, 0xA2}, DataTypeInt, false}, + ElementProjection: elementDef{[]byte{0x76, 0x70}, DataTypeMaster, false}, + ElementProjectionType: elementDef{[]byte{0x76, 0x71}, DataTypeUInt, false}, + ElementProjectionPrivate: elementDef{[]byte{0x76, 0x72}, DataTypeBinary, false}, + ElementProjectionPoseYaw: elementDef{[]byte{0x76, 0x73}, DataTypeFloat, false}, + ElementProjectionPosePitch: elementDef{[]byte{0x76, 0x74}, DataTypeFloat, false}, + ElementProjectionPoseRoll: elementDef{[]byte{0x76, 0x75}, DataTypeFloat, false}, + ElementOutputSamplingFrequency: elementDef{[]byte{0x78, 0xB5}, DataTypeFloat, false}, + ElementTitle: elementDef{[]byte{0x7B, 0xA9}, DataTypeString, false}, + ElementChapterDisplay: elementDef{[]byte{0x80}, DataTypeMaster, false}, + ElementTrackType: elementDef{[]byte{0x83}, DataTypeUInt, false}, + ElementChapString: elementDef{[]byte{0x85}, DataTypeString, false}, + ElementCodecID: elementDef{[]byte{0x86}, DataTypeString, false}, + ElementFlagDefault: elementDef{[]byte{0x88}, DataTypeUInt, false}, + ElementChapterTrackUID: elementDef{[]byte{0x89}, DataTypeUInt, false}, + ElementSlices: elementDef{[]byte{0x8E}, DataTypeMaster, false}, + ElementChapterTrack: elementDef{[]byte{0x8F}, DataTypeMaster, false}, + ElementChapterTimeStart: elementDef{[]byte{0x91}, DataTypeUInt, false}, + ElementChapterTimeEnd: elementDef{[]byte{0x92}, DataTypeUInt, false}, + ElementCueRefTime: elementDef{[]byte{0x96}, DataTypeUInt, false}, + ElementChapterFlagHidden: elementDef{[]byte{0x98}, DataTypeUInt, false}, + ElementFlagInterlaced: elementDef{[]byte{0x9A}, DataTypeUInt, false}, + ElementBlockDuration: elementDef{[]byte{0x9B}, DataTypeUInt, false}, + ElementFlagLacing: elementDef{[]byte{0x9C}, DataTypeUInt, false}, + ElementFieldOrder: elementDef{[]byte{0x9D}, DataTypeUInt, false}, + ElementChannels: elementDef{[]byte{0x9F}, DataTypeUInt, false}, + ElementBlockGroup: elementDef{[]byte{0xA0}, DataTypeMaster, false}, + ElementBlock: elementDef{[]byte{0xA1}, DataTypeBlock, false}, + ElementSimpleBlock: elementDef{[]byte{0xA3}, DataTypeBlock, false}, + ElementCodecState: elementDef{[]byte{0xA4}, DataTypeBinary, false}, + ElementBlockAdditional: elementDef{[]byte{0xA5}, DataTypeBinary, false}, + ElementBlockMore: elementDef{[]byte{0xA6}, DataTypeMaster, false}, + ElementPosition: elementDef{[]byte{0xA7}, DataTypeUInt, false}, + ElementCodecDecodeAll: elementDef{[]byte{0xAA}, DataTypeUInt, false}, + ElementPrevSize: elementDef{[]byte{0xAB}, DataTypeUInt, false}, + ElementTrackEntry: elementDef{[]byte{0xAE}, DataTypeMaster, false}, + ElementPixelWidth: elementDef{[]byte{0xB0}, DataTypeUInt, false}, + ElementCueDuration: elementDef{[]byte{0xB2}, DataTypeUInt, false}, + ElementCueTime: elementDef{[]byte{0xB3}, DataTypeUInt, false}, + ElementSamplingFrequency: elementDef{[]byte{0xB5}, DataTypeFloat, false}, + ElementChapterAtom: elementDef{[]byte{0xB6}, DataTypeMaster, false}, + ElementCueTrackPositions: elementDef{[]byte{0xB7}, DataTypeMaster, false}, + ElementFlagEnabled: elementDef{[]byte{0xB9}, DataTypeUInt, false}, + ElementPixelHeight: elementDef{[]byte{0xBA}, DataTypeUInt, false}, + ElementCuePoint: elementDef{[]byte{0xBB}, DataTypeMaster, false}, + ElementCRC32: elementDef{[]byte{0xBF}, DataTypeBinary, false}, + ElementLaceNumber: elementDef{[]byte{0xCC}, DataTypeUInt, false}, + ElementTrackNumber: elementDef{[]byte{0xD7}, DataTypeUInt, false}, + ElementCueReference: elementDef{[]byte{0xDB}, DataTypeMaster, false}, + ElementVideo: elementDef{[]byte{0xE0}, DataTypeMaster, false}, + ElementAudio: elementDef{[]byte{0xE1}, DataTypeMaster, false}, + ElementTrackOperation: elementDef{[]byte{0xE2}, DataTypeMaster, false}, + ElementTrackCombinePlanes: elementDef{[]byte{0xE3}, DataTypeMaster, false}, + ElementTrackPlane: elementDef{[]byte{0xE4}, DataTypeMaster, false}, + ElementTrackPlaneUID: elementDef{[]byte{0xE5}, DataTypeUInt, false}, + ElementTrackPlaneType: elementDef{[]byte{0xE6}, DataTypeUInt, false}, + ElementTimecode: elementDef{[]byte{0xE7}, DataTypeUInt, false}, + ElementTimeSlice: elementDef{[]byte{0xE8}, DataTypeMaster, false}, + ElementTrackJoinBlocks: elementDef{[]byte{0xE9}, DataTypeMaster, false}, + ElementCueCodecState: elementDef{[]byte{0xEA}, DataTypeUInt, false}, + ElementVoid: elementDef{[]byte{0xEC}, DataTypeBinary, false}, + ElementTrackJoinUID: elementDef{[]byte{0xED}, DataTypeUInt, false}, + ElementBlockAddID: elementDef{[]byte{0xEE}, DataTypeUInt, false}, + ElementCueRelativePosition: elementDef{[]byte{0xF0}, DataTypeUInt, false}, + ElementCueClusterPosition: elementDef{[]byte{0xF1}, DataTypeUInt, false}, + ElementCueTrack: elementDef{[]byte{0xF7}, DataTypeUInt, false}, + ElementReferencePriority: elementDef{[]byte{0xFA}, DataTypeUInt, false}, + ElementReferenceBlock: elementDef{[]byte{0xFB}, DataTypeInt, false}, +} + +type elementRevTable map[uint32]element +type element struct { + e ElementType + t DataType + top bool +} + +var revTable elementRevTable + +func init() { + revTable = make(elementRevTable) + initReverseLookupTable(revTable, table) +} + +func initReverseLookupTable(revTb elementRevTable, tb elementTable) { + vd := &valueDecoder{} + for k, v := range tb { + e, _, err := vd.readVUInt(bytes.NewBuffer(v.b)) + if err != nil { + panic(err) + } + revTb[uint32(e)] = element{e: k, t: v.t, top: v.top} + } +} diff --git a/pkg/ebml-go/elementtype.go b/pkg/ebml-go/elementtype.go new file mode 100644 index 0000000..6c40f74 --- /dev/null +++ b/pkg/ebml-go/elementtype.go @@ -0,0 +1,550 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ebml + +import ( + "errors" +) + +// ErrUnknownElementName means that a element name is not found in the ElementType list. +var ErrUnknownElementName = errors.New("unknown element name") + +// ElementType represents EBML Element type. +type ElementType int + +// EBML Element types. +const ( + ElementInvalid ElementType = iota + + ElementEBML + ElementEBMLVersion + ElementEBMLReadVersion + ElementEBMLMaxIDLength + ElementEBMLMaxSizeLength + ElementEBMLDocType + ElementEBMLDocTypeVersion + ElementEBMLDocTypeReadVersion + + ElementCRC32 + ElementVoid + ElementSegment + + ElementSeekHead + ElementSeek + ElementSeekID + ElementSeekPosition + + ElementInfo + ElementSegmentUID + ElementSegmentFilename + ElementPrevUID + ElementPrevFilename + ElementNextUID + ElementNextFilename + ElementSegmentFamily + ElementChapterTranslate + ElementChapterTranslateEditionUID + ElementChapterTranslateCodec + ElementChapterTranslateID + ElementTimestampScale + ElementDuration + ElementDateUTC + ElementTitle + ElementMuxingApp + ElementWritingApp + + ElementCluster + ElementTimestamp + ElementSilentTracks + ElementSilentTrackNumber + ElementPosition + ElementPrevSize + ElementSimpleBlock + ElementBlockGroup + ElementBlock + ElementBlockAdditions + ElementBlockMore + ElementBlockAddID + ElementBlockAdditional + ElementBlockDuration + ElementReferencePriority + ElementReferenceBlock + ElementCodecState + ElementDiscardPadding + ElementSlices + // Deprecated: Dropped in v2 + ElementTimeSlice + // Deprecated: Dropped in v2 + ElementLaceNumber + + ElementTracks + ElementTrackEntry + ElementTrackNumber + ElementTrackUID + ElementTrackType + ElementFlagEnabled + ElementFlagDefault + ElementFlagForced + ElementFlagLacing + ElementMinCache + ElementMaxCache + ElementDefaultDuration + ElementDefaultDecodedFieldDuration + // Deprecated: Dropped in v4 + ElementTrackTimestampScale + ElementMaxBlockAdditionID + ElementBlockAdditionMapping + ElementBlockAddIDValue + ElementBlockAddIDName + ElementBlockAddIDType + ElementBlockAddIDExtraData + ElementName + ElementLanguage + ElementLanguageIETF + ElementCodecID + ElementCodecPrivate + ElementCodecName + // Deprecated: Dropped in v4 + ElementAttachmentLink + ElementCodecDecodeAll + ElementTrackOverlay + ElementCodecDelay + ElementSeekPreRoll + ElementTrackTranslate + ElementTrackTranslateEditionUID + ElementTrackTranslateCodec + ElementTrackTranslateTrackID + ElementVideo + ElementFlagInterlaced + ElementFieldOrder + ElementStereoMode + ElementAlphaMode + ElementPixelWidth + ElementPixelHeight + ElementPixelCropBottom + ElementPixelCropTop + ElementPixelCropLeft + ElementPixelCropRight + ElementDisplayWidth + ElementDisplayHeight + ElementDisplayUnit + ElementAspectRatioType + ElementColourSpace + ElementColour + ElementMatrixCoefficients + ElementBitsPerChannel + ElementChromaSubsamplingHorz + ElementChromaSubsamplingVert + ElementCbSubsamplingHorz + ElementCbSubsamplingVert + ElementChromaSitingHorz + ElementChromaSitingVert + ElementRange + ElementTransferCharacteristics + ElementPrimaries + ElementMaxCLL + ElementMaxFALL + ElementMasteringMetadata + ElementPrimaryRChromaticityX + ElementPrimaryRChromaticityY + ElementPrimaryGChromaticityX + ElementPrimaryGChromaticityY + ElementPrimaryBChromaticityX + ElementPrimaryBChromaticityY + ElementWhitePointChromaticityX + ElementWhitePointChromaticityY + ElementLuminanceMax + ElementLuminanceMin + ElementProjection + ElementProjectionType + ElementProjectionPrivate + ElementProjectionPoseYaw + ElementProjectionPosePitch + ElementProjectionPoseRoll + ElementAudio + ElementSamplingFrequency + ElementOutputSamplingFrequency + ElementChannels + ElementBitDepth + ElementTrackOperation + ElementTrackCombinePlanes + ElementTrackPlane + ElementTrackPlaneUID + ElementTrackPlaneType + ElementTrackJoinBlocks + ElementTrackJoinUID + ElementContentEncodings + ElementContentEncoding + ElementContentEncodingOrder + ElementContentEncodingScope + ElementContentEncodingType + ElementContentCompression + ElementContentCompAlgo + ElementContentCompSettings + ElementContentEncryption + ElementContentEncAlgo + ElementContentEncKeyID + ElementContentEncAESSettings + ElementAESSettingsCipherMode + ElementContentSignature + ElementContentSigKeyID + ElementContentSigAlgo + ElementContentSigHashAlgo + + ElementCues + ElementCuePoint + ElementCueTime + ElementCueTrackPositions + ElementCueTrack + ElementCueClusterPosition + ElementCueRelativePosition + ElementCueDuration + ElementCueBlockNumber + ElementCueCodecState + ElementCueReference + ElementCueRefTime + + ElementAttachments + ElementAttachedFile + ElementFileDescription + ElementFileName + ElementFileMimeType + ElementFileData + ElementFileUID + + ElementChapters + ElementEditionEntry + ElementEditionUID + ElementEditionFlagHidden + ElementEditionFlagDefault + ElementEditionFlagOrdered + ElementChapterAtom + ElementChapterUID + ElementChapterStringUID + ElementChapterTimeStart + ElementChapterTimeEnd + ElementChapterFlagHidden + ElementChapterFlagEnabled + ElementChapterSegmentUID + ElementChapterSegmentEditionUID + ElementChapterPhysicalEquiv + ElementChapterTrack + ElementChapterTrackUID + ElementChapterDisplay + ElementChapString + ElementChapLanguage + ElementChapLanguageIETF + ElementChapCountry + ElementChapProcess + ElementChapProcessCodecID + ElementChapProcessPrivate + ElementChapProcessCommand + ElementChapProcessTime + ElementChapProcessData + + ElementTags + ElementTag + ElementTargets + ElementTargetTypeValue + ElementTargetType + ElementTagTrackUID + ElementTagEditionUID + ElementTagChapterUID + ElementTagAttachmentUID + ElementSimpleTag + ElementTagName + ElementTagLanguage + ElementTagLanguageIETF + ElementTagDefault + ElementTagString + ElementTagBinary + + elementMax +) + +// WebM aliases +const ( + ElementTimecodeScale = ElementTimestampScale + ElementTimecode = ElementTimestamp +) + +var elementTypeName = map[ElementType]string{ + ElementEBML: "EBML", + ElementEBMLVersion: "EBMLVersion", + ElementEBMLReadVersion: "EBMLReadVersion", + ElementEBMLMaxIDLength: "EBMLMaxIDLength", + ElementEBMLMaxSizeLength: "EBMLMaxSizeLength", + ElementEBMLDocType: "EBMLDocType", + ElementEBMLDocTypeVersion: "EBMLDocTypeVersion", + ElementEBMLDocTypeReadVersion: "EBMLDocTypeReadVersion", + ElementCRC32: "CRC32", + ElementVoid: "Void", + ElementSegment: "Segment", + ElementSeekHead: "SeekHead", + ElementSeek: "Seek", + ElementSeekID: "SeekID", + ElementSeekPosition: "SeekPosition", + ElementInfo: "Info", + ElementSegmentUID: "SegmentUID", + ElementSegmentFilename: "SegmentFilename", + ElementPrevUID: "PrevUID", + ElementPrevFilename: "PrevFilename", + ElementNextUID: "NextUID", + ElementNextFilename: "NextFilename", + ElementSegmentFamily: "SegmentFamily", + ElementChapterTranslate: "ChapterTranslate", + ElementChapterTranslateEditionUID: "ChapterTranslateEditionUID", + ElementChapterTranslateCodec: "ChapterTranslateCodec", + ElementChapterTranslateID: "ChapterTranslateID", + ElementTimestampScale: "TimestampScale", + ElementDuration: "Duration", + ElementDateUTC: "DateUTC", + ElementTitle: "Title", + ElementMuxingApp: "MuxingApp", + ElementWritingApp: "WritingApp", + ElementCluster: "Cluster", + ElementTimestamp: "Timestamp", + ElementSilentTracks: "SilentTracks", + ElementSilentTrackNumber: "SilentTrackNumber", + ElementPosition: "Position", + ElementPrevSize: "PrevSize", + ElementSimpleBlock: "SimpleBlock", + ElementBlockGroup: "BlockGroup", + ElementBlock: "Block", + ElementBlockAdditions: "BlockAdditions", + ElementBlockMore: "BlockMore", + ElementBlockAddID: "BlockAddID", + ElementBlockAdditional: "BlockAdditional", + ElementBlockDuration: "BlockDuration", + ElementReferencePriority: "ReferencePriority", + ElementReferenceBlock: "ReferenceBlock", + ElementCodecState: "CodecState", + ElementDiscardPadding: "DiscardPadding", + ElementSlices: "Slices", + ElementTimeSlice: "TimeSlice", + ElementLaceNumber: "LaceNumber", + ElementTracks: "Tracks", + ElementTrackEntry: "TrackEntry", + ElementTrackNumber: "TrackNumber", + ElementTrackUID: "TrackUID", + ElementTrackType: "TrackType", + ElementFlagEnabled: "FlagEnabled", + ElementFlagDefault: "FlagDefault", + ElementFlagForced: "FlagForced", + ElementFlagLacing: "FlagLacing", + ElementMinCache: "MinCache", + ElementMaxCache: "MaxCache", + ElementDefaultDuration: "DefaultDuration", + ElementDefaultDecodedFieldDuration: "DefaultDecodedFieldDuration", + ElementTrackTimestampScale: "TrackTimestampScale", + ElementMaxBlockAdditionID: "MaxBlockAdditionID", + ElementBlockAdditionMapping: "BlockAdditionMapping", + ElementBlockAddIDValue: "BlockAddIDValue", + ElementBlockAddIDName: "BlockAddIDName", + ElementBlockAddIDType: "BlockAddIDType", + ElementBlockAddIDExtraData: "BlockAddIDExtraData", + ElementName: "Name", + ElementLanguage: "Language", + ElementLanguageIETF: "LanguageIETF", + ElementCodecID: "CodecID", + ElementCodecPrivate: "CodecPrivate", + ElementCodecName: "CodecName", + ElementAttachmentLink: "AttachmentLink", + ElementCodecDecodeAll: "CodecDecodeAll", + ElementTrackOverlay: "TrackOverlay", + ElementCodecDelay: "CodecDelay", + ElementSeekPreRoll: "SeekPreRoll", + ElementTrackTranslate: "TrackTranslate", + ElementTrackTranslateEditionUID: "TrackTranslateEditionUID", + ElementTrackTranslateCodec: "TrackTranslateCodec", + ElementTrackTranslateTrackID: "TrackTranslateTrackID", + ElementVideo: "Video", + ElementFlagInterlaced: "FlagInterlaced", + ElementFieldOrder: "FieldOrder", + ElementStereoMode: "StereoMode", + ElementAlphaMode: "AlphaMode", + ElementPixelWidth: "PixelWidth", + ElementPixelHeight: "PixelHeight", + ElementPixelCropBottom: "PixelCropBottom", + ElementPixelCropTop: "PixelCropTop", + ElementPixelCropLeft: "PixelCropLeft", + ElementPixelCropRight: "PixelCropRight", + ElementDisplayWidth: "DisplayWidth", + ElementDisplayHeight: "DisplayHeight", + ElementDisplayUnit: "DisplayUnit", + ElementAspectRatioType: "AspectRatioType", + ElementColourSpace: "ColourSpace", + ElementColour: "Colour", + ElementMatrixCoefficients: "MatrixCoefficients", + ElementBitsPerChannel: "BitsPerChannel", + ElementChromaSubsamplingHorz: "ChromaSubsamplingHorz", + ElementChromaSubsamplingVert: "ChromaSubsamplingVert", + ElementCbSubsamplingHorz: "CbSubsamplingHorz", + ElementCbSubsamplingVert: "CbSubsamplingVert", + ElementChromaSitingHorz: "ChromaSitingHorz", + ElementChromaSitingVert: "ChromaSitingVert", + ElementRange: "Range", + ElementTransferCharacteristics: "TransferCharacteristics", + ElementPrimaries: "Primaries", + ElementMaxCLL: "MaxCLL", + ElementMaxFALL: "MaxFALL", + ElementMasteringMetadata: "MasteringMetadata", + ElementPrimaryRChromaticityX: "PrimaryRChromaticityX", + ElementPrimaryRChromaticityY: "PrimaryRChromaticityY", + ElementPrimaryGChromaticityX: "PrimaryGChromaticityX", + ElementPrimaryGChromaticityY: "PrimaryGChromaticityY", + ElementPrimaryBChromaticityX: "PrimaryBChromaticityX", + ElementPrimaryBChromaticityY: "PrimaryBChromaticityY", + ElementWhitePointChromaticityX: "WhitePointChromaticityX", + ElementWhitePointChromaticityY: "WhitePointChromaticityY", + ElementLuminanceMax: "LuminanceMax", + ElementLuminanceMin: "LuminanceMin", + ElementProjection: "Projection", + ElementProjectionType: "ProjectionType", + ElementProjectionPrivate: "ProjectionPrivate", + ElementProjectionPoseYaw: "ProjectionPoseYaw", + ElementProjectionPosePitch: "ProjectionPosePitch", + ElementProjectionPoseRoll: "ProjectionPoseRoll", + ElementAudio: "Audio", + ElementSamplingFrequency: "SamplingFrequency", + ElementOutputSamplingFrequency: "OutputSamplingFrequency", + ElementChannels: "Channels", + ElementBitDepth: "BitDepth", + ElementTrackOperation: "TrackOperation", + ElementTrackCombinePlanes: "TrackCombinePlanes", + ElementTrackPlane: "TrackPlane", + ElementTrackPlaneUID: "TrackPlaneUID", + ElementTrackPlaneType: "TrackPlaneType", + ElementTrackJoinBlocks: "TrackJoinBlocks", + ElementTrackJoinUID: "TrackJoinUID", + ElementContentEncodings: "ContentEncodings", + ElementContentEncoding: "ContentEncoding", + ElementContentEncodingOrder: "ContentEncodingOrder", + ElementContentEncodingScope: "ContentEncodingScope", + ElementContentEncodingType: "ContentEncodingType", + ElementContentCompression: "ContentCompression", + ElementContentCompAlgo: "ContentCompAlgo", + ElementContentCompSettings: "ContentCompSettings", + ElementContentEncryption: "ContentEncryption", + ElementContentEncAlgo: "ContentEncAlgo", + ElementContentEncKeyID: "ContentEncKeyID", + ElementContentEncAESSettings: "ContentEncAESSettings", + ElementAESSettingsCipherMode: "AESSettingsCipherMode", + ElementContentSignature: "ContentSignature", + ElementContentSigKeyID: "ContentSigKeyID", + ElementContentSigAlgo: "ContentSigAlgo", + ElementContentSigHashAlgo: "ContentSigHashAlgo", + ElementCues: "Cues", + ElementCuePoint: "CuePoint", + ElementCueTime: "CueTime", + ElementCueTrackPositions: "CueTrackPositions", + ElementCueTrack: "CueTrack", + ElementCueClusterPosition: "CueClusterPosition", + ElementCueRelativePosition: "CueRelativePosition", + ElementCueDuration: "CueDuration", + ElementCueBlockNumber: "CueBlockNumber", + ElementCueCodecState: "CueCodecState", + ElementCueReference: "CueReference", + ElementCueRefTime: "CueRefTime", + ElementAttachments: "Attachments", + ElementAttachedFile: "AttachedFile", + ElementFileDescription: "FileDescription", + ElementFileName: "FileName", + ElementFileMimeType: "FileMimeType", + ElementFileData: "FileData", + ElementFileUID: "FileUID", + ElementChapters: "Chapters", + ElementEditionEntry: "EditionEntry", + ElementEditionUID: "EditionUID", + ElementEditionFlagHidden: "EditionFlagHidden", + ElementEditionFlagDefault: "EditionFlagDefault", + ElementEditionFlagOrdered: "EditionFlagOrdered", + ElementChapterAtom: "ChapterAtom", + ElementChapterUID: "ChapterUID", + ElementChapterStringUID: "ChapterStringUID", + ElementChapterTimeStart: "ChapterTimeStart", + ElementChapterTimeEnd: "ChapterTimeEnd", + ElementChapterFlagHidden: "ChapterFlagHidden", + ElementChapterFlagEnabled: "ChapterFlagEnabled", + ElementChapterSegmentUID: "ChapterSegmentUID", + ElementChapterSegmentEditionUID: "ChapterSegmentEditionUID", + ElementChapterPhysicalEquiv: "ChapterPhysicalEquiv", + ElementChapterTrack: "ChapterTrack", + ElementChapterTrackUID: "ChapterTrackUID", + ElementChapterDisplay: "ChapterDisplay", + ElementChapString: "ChapString", + ElementChapLanguage: "ChapLanguage", + ElementChapLanguageIETF: "ChapLanguageIETF", + ElementChapCountry: "ChapCountry", + ElementChapProcess: "ChapProcess", + ElementChapProcessCodecID: "ChapProcessCodecID", + ElementChapProcessPrivate: "ChapProcessPrivate", + ElementChapProcessCommand: "ChapProcessCommand", + ElementChapProcessTime: "ChapProcessTime", + ElementChapProcessData: "ChapProcessData", + ElementTags: "Tags", + ElementTag: "Tag", + ElementTargets: "Targets", + ElementTargetTypeValue: "TargetTypeValue", + ElementTargetType: "TargetType", + ElementTagTrackUID: "TagTrackUID", + ElementTagEditionUID: "TagEditionUID", + ElementTagChapterUID: "TagChapterUID", + ElementTagAttachmentUID: "TagAttachmentUID", + ElementSimpleTag: "SimpleTag", + ElementTagName: "TagName", + ElementTagLanguage: "TagLanguage", + ElementTagLanguageIETF: "TagLanguageIETF", + ElementTagDefault: "TagDefault", + ElementTagString: "TagString", + ElementTagBinary: "TagBinary", +} + +func (i ElementType) String() string { + if name, ok := elementTypeName[i]; ok { + return name + } + return "unknown" +} + +// Bytes returns []byte representation of the element ID. +func (i ElementType) Bytes() []byte { + return table[i].b +} + +// DataType returns DataType of the element. +func (i ElementType) DataType() DataType { + return table[i].t +} + +var elementNameType map[string]ElementType + +// ElementTypeFromString converts string to ElementType. +func ElementTypeFromString(s string) (ElementType, error) { + if t, ok := elementNameType[s]; ok { + return t, nil + } + return 0, wrapErrorf(ErrUnknownElementName, "parsing \"%s\"", s) +} + +func init() { + elementNameType = make(map[string]ElementType) + for t, name := range elementTypeName { + elementNameType[name] = t + } + // WebM aliases + elementNameType["TimecodeScale"] = ElementTimecodeScale + elementNameType["Timecode"] = ElementTimecode +} diff --git a/pkg/ebml-go/elementtype_test.go b/pkg/ebml-go/elementtype_test.go new file mode 100644 index 0000000..f337355 --- /dev/null +++ b/pkg/ebml-go/elementtype_test.go @@ -0,0 +1,68 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ebml + +import ( + "bytes" + "io" + "testing" + + "github.com/at-wat/ebml-go/internal/errs" +) + +func TestElementType_Roundtrip(t *testing.T) { + for e := ElementInvalid + 1; e < elementMax; e++ { + s := e.String() + if el, err := ElementTypeFromString(s); err != nil { + t.Errorf("Failed to get ElementType from string: '%v'", err) + } else if e != el { + t.Errorf("Failed to roundtrip ElementType %d and string", e) + } + } + if elementMax.String() != "unknown" { + t.Errorf("Invalid ElementType string should be 'unknown', got '%s'", elementMax.String()) + } +} + +func TestElementType_Bytes(t *testing.T) { + expected := []byte{0x18, 0x53, 0x80, 0x67} + + if !bytes.Equal(ElementSegment.Bytes(), expected) { + t.Errorf("Expected bytes: '%v', got: '%v'", expected, ElementSegment.Bytes()) + } + if ElementSegment.DataType() != DataTypeMaster { + t.Errorf("Expected DataType: %s, got: %s", DataTypeMaster, ElementSegment.DataType()) + } +} + +func TestElementType_InitReverseLookupTable(t *testing.T) { + defer func() { + err := recover() + switch v := err.(type) { + case error: + if !errs.Is(v, io.ErrUnexpectedEOF) { + t.Errorf("Expected initReverseLookupTable panic: '%v', got: '%v'", io.ErrUnexpectedEOF, v) + } + default: + t.Errorf("initReverseLookupTable paniced with unexpected type %T", v) + } + }() + + revTb := make(elementRevTable) + initReverseLookupTable(revTb, elementTable{ + ElementType(0): elementDef{}, // empty bytes representation + }) + t.Fatal("initReverseLookupTable must panic if elementTable is broken.") +} diff --git a/pkg/ebml-go/error.go b/pkg/ebml-go/error.go new file mode 100644 index 0000000..99b496e --- /dev/null +++ b/pkg/ebml-go/error.go @@ -0,0 +1,87 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ebml + +import ( + "fmt" + "reflect" +) + +// Error records a failed parsing. +type Error struct { + Err error + Failure string +} + +func (e *Error) Error() string { + // TODO: migrate to fmt.Sprintf %w once Go1.12 reaches EOL. + return e.Failure + ": " + e.Err.Error() +} + +// Unwrap returns the reason of the failure. +// This is for Go1.13 error unwrapping. +func (e *Error) Unwrap() error { + return e.Err +} + +// Is reports whether chained error contains target. +// This is for Go1.13 error unwrapping. +func (e *Error) Is(target error) bool { + err := e.Err + + switch target { + case e: + return true + case nil: + return err == nil + } + for { + switch err { + case nil: + return false + case target: + return true + } + x, ok := err.(interface{ Unwrap() error }) + if !ok { + // Some stdlibs haven't have error unwrapper yet. + // Check err.Err field if exposed. + if reflect.TypeOf(err).Kind() == reflect.Ptr { + e := reflect.ValueOf(err).Elem().FieldByName("Err") + if e.IsValid() { + e2, ok := e.Interface().(error) + if !ok { + return false + } + err = e2 + continue + } + } + return false + } + err = x.Unwrap() + } +} + +func wrapError(err error, failure string) error { + return &Error{ + Failure: failure, + Err: err, + } +} + +func wrapErrorf(err error, failureFmt string, v ...interface{}) error { + return wrapError(err, fmt.Sprintf(failureFmt, v...)) +} diff --git a/pkg/ebml-go/error_test.go b/pkg/ebml-go/error_test.go new file mode 100644 index 0000000..24154ac --- /dev/null +++ b/pkg/ebml-go/error_test.go @@ -0,0 +1,87 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ebml + +import ( + "errors" + "testing" + + "github.com/at-wat/ebml-go/internal/errs" +) + +type dummyError struct { + Err error +} + +func (e *dummyError) Error() string { + return e.Err.Error() +} + +func TestError(t *testing.T) { + errBase := errors.New("an error") + errOther := errors.New("an another error") + errChained := wrapErrorf(errBase, "info") + errDoubleChained := wrapErrorf(errChained, "info") + errChainedNil := wrapErrorf(nil, "info") + errChainedOther := wrapErrorf(errOther, "info") + err112Chained := wrapErrorf(&dummyError{errBase}, "info") + err112Nil := wrapErrorf(&dummyError{nil}, "info") + errStr := "info: an error" + + t.Run("ErrorsIs", func(t *testing.T) { + if !errs.Is(errChained, errBase) { + t.Errorf("Wrapped error '%v' doesn't chain '%v'", errChained, errBase) + } + }) + + t.Run("Is", func(t *testing.T) { + if !errChained.(*Error).Is(errChained) { + t.Errorf("Wrapped error '%v' doesn't match its-self", errChained) + } + if !errChained.(*Error).Is(errBase) { + t.Errorf("Wrapped error '%v' doesn't match '%v'", errChained, errBase) + } + if !errDoubleChained.(*Error).Is(errBase) { + t.Errorf("Wrapped error '%v' doesn't match '%v'", errDoubleChained, errBase) + } + if !err112Chained.(*Error).Is(errBase) { + t.Errorf("Wrapped error '%v' doesn't match '%v'", + err112Chained, errBase) + } + if !errChainedNil.(*Error).Is(nil) { + t.Errorf("Nil chained error '%v' doesn't match 'nil'", errChainedNil) + } + + if errChainedNil.(*Error).Is(errBase) { + t.Errorf("Wrapped error '%v' unexpectedly matched '%v'", + errChainedNil, errBase) + } + if errChainedOther.(*Error).Is(errBase) { + t.Errorf("Wrapped error '%v' unexpectedly matched '%v'", + errChainedOther, errBase) + } + if err112Nil.(*Error).Is(errBase) { + t.Errorf("Wrapped error '%v' unexpectedly matched '%v'", + errChainedOther, errBase) + } + }) + + if errChained.Error() != errStr { + t.Errorf("Error string expected: %s, got: %s", errStr, errChained.Error()) + } + if errChained.(*Error).Unwrap() != errBase { + t.Errorf("Unwrapped error expected: %s, got: %s", errBase, errChained.(*Error).Unwrap()) + } +} diff --git a/pkg/ebml-go/examples/README.md b/pkg/ebml-go/examples/README.md new file mode 100644 index 0000000..6bec6c3 --- /dev/null +++ b/pkg/ebml-go/examples/README.md @@ -0,0 +1,34 @@ +# ebml-go examples + +## rtp-to-webm + +Receive RTP VP8 stream UDP packets and pack it to WebM file. + +1. Run the following command. + ```shell + $ cd rtp-to-webm + $ go build . + $ ./rtp-to-webm + ``` +2. Send RTP stream to `./rtp-to-webm` using GStreamer. + ```shell + $ gst-launch-1.0 videotestsrc \ + ! video/x-raw,width=320,height=240,framerate=30/1 \ + ! vp8enc target-bitrate=4000 \ + ! rtpvp8pay ! udpsink host=localhost port=4000 + ``` +3. Check out `test.webm` generated at the current directory. + + +## webm-roundtrip + +Read WebM file, parse, and write back to the file. + +1. Run the following command to read `sample.webm`. + ```shell + $ cd webm-roundtrip + $ go build . + $ ./webm-roundtrip + ``` +2. Contents (EBML document) of the file is shown to the stdout. +3. Check out `copy.webm` generated at the current directory. It should be playable and seekable as same as `sample.webm`. diff --git a/pkg/ebml-go/examples/rtp-to-webm/.gitignore b/pkg/ebml-go/examples/rtp-to-webm/.gitignore new file mode 100644 index 0000000..1bfce3d --- /dev/null +++ b/pkg/ebml-go/examples/rtp-to-webm/.gitignore @@ -0,0 +1,2 @@ +rtp-to-webm +*.webm diff --git a/pkg/ebml-go/examples/rtp-to-webm/main.go b/pkg/ebml-go/examples/rtp-to-webm/main.go new file mode 100644 index 0000000..f77be69 --- /dev/null +++ b/pkg/ebml-go/examples/rtp-to-webm/main.go @@ -0,0 +1,140 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/binary" + "fmt" + "net" + "os" + "os/signal" + + "github.com/at-wat/ebml-go/webm" +) + +func main() { + w, err := os.OpenFile("test.webm", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) + if err != nil { + panic(err) + } + + ws, err := webm.NewSimpleBlockWriter(w, + []webm.TrackEntry{ + { + Name: "Video", + TrackNumber: 1, + TrackUID: 12345, + CodecID: "V_VP8", + TrackType: 1, + DefaultDuration: 33333333, + Video: &webm.Video{ + PixelWidth: 320, + PixelHeight: 240, + }, + }, + }) + if err != nil { + panic(err) + } + + fmt.Print("Run following command to send RTP stream to this example:\n" + + "$ gst-launch-1.0 videotestsrc" + + " ! video/x-raw,width=320,height=240,framerate=30/1" + + " ! vp8enc target-bitrate=4000" + + " ! rtpvp8pay ! udpsink host=localhost port=4000\n\n" + + "Waiting first keyframe...\n") + + // Listen UDP RTP packets + addr, err := net.ResolveUDPAddr("udp", ":4000") + if err != nil { + panic(err) + } + pc, err := net.ListenUDP("udp", addr) + if err != nil { + panic(err) + } + + closed := make(chan os.Signal, 1) + signal.Notify(closed, os.Interrupt) + go func() { + <-closed + pc.Close() + }() + + var frame []byte + var keyframe bool + var keyframeCnt int + var tcRawLast int64 = -1 + var tcRawBase int64 + buffer := make([]byte, 1522) + + for { + n, _, err := pc.ReadFrom(buffer) + if err != nil { + fmt.Println(err) + break + } + if n < 14 { + fmt.Print("RTP packet size is too small.\n") + continue + } + + // RTP descriptor and header must be fully parsed in the real application. + vp8Desc := buffer[12] + + // *Extended control bits present flag* is not supported in this example + if vp8Desc&0x80 != 0 { + panic("Incoming VP8 payload descriptor has extended fields which is not supported by this example!") + } + + // *Start of VP8 partition flag* + if vp8Desc&0x10 != 0 { + if keyframe { + keyframeCnt++ + } + if len(frame) > 0 && keyframeCnt > 0 { + tcRaw := int64(binary.BigEndian.Uint32(buffer[4:8])) + if tcRawLast == -1 { + tcRawLast = tcRaw + } + if tcRaw < 0x10000 && tcRawLast > 0xFFFFFFFF-0x10000 { + // counter overflow + tcRawBase += 0x100000000 + } else if tcRawLast < 0x10000 && tcRaw > 0xFFFFFFFF-0x10000 { + // counter underflow + tcRawBase -= 0x100000000 + } + tcRawLast = tcRaw + + tc := (tcRaw + tcRawBase) / 90 // VP8 timestamp rate is 90000. + fmt.Printf("RTP frame received. (len: %d, timestamp: %d, keyframe: %v)\n", len(frame), tc, keyframe) + ws[0].Write(keyframe, int64(tc), frame) + } + frame = []byte{} + keyframe = false + + // RTP header 12 bytes, VP8 payload descriptor 1 byte. + vp8Header := buffer[13] + if vp8Header&0x01 == 0 { + keyframe = true + } + } + + frame = append(frame, buffer[13:n]...) + } + + fmt.Printf("\nFinalizing webm...\n") + ws[0].Close() +} diff --git a/pkg/ebml-go/examples/webm-roundtrip/.gitignore b/pkg/ebml-go/examples/webm-roundtrip/.gitignore new file mode 100644 index 0000000..73030ec --- /dev/null +++ b/pkg/ebml-go/examples/webm-roundtrip/.gitignore @@ -0,0 +1,2 @@ +webm-roundtrip +copy.webm diff --git a/pkg/ebml-go/examples/webm-roundtrip/main.go b/pkg/ebml-go/examples/webm-roundtrip/main.go new file mode 100644 index 0000000..290c929 --- /dev/null +++ b/pkg/ebml-go/examples/webm-roundtrip/main.go @@ -0,0 +1,56 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/at-wat/ebml-go" + "github.com/at-wat/ebml-go/webm" +) + +func main() { + r, err := os.Open("sample.webm") + if err != nil { + panic(err) + } + defer r.Close() + + var ret struct { + Header webm.EBMLHeader `ebml:"EBML"` + Segment webm.Segment `ebml:"Segment"` + } + if err := ebml.Unmarshal(r, &ret); err != nil { + fmt.Printf("error: %v\n", err) + return + } + j, err := json.MarshalIndent(ret, "", " ") + if err != nil { + fmt.Printf("error: %v\n", err) + return + } + fmt.Printf("%s\n", string(j)) + + w, err := os.OpenFile("copy.webm", os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + panic(err) + } + defer w.Close() + if err := ebml.Marshal(&ret, w); err != nil { + panic(err) + } +} diff --git a/pkg/ebml-go/examples/webm-roundtrip/sample.webm b/pkg/ebml-go/examples/webm-roundtrip/sample.webm new file mode 100644 index 0000000000000000000000000000000000000000..4d573c522a4e638dc3d61f8f92162ee3d9ad54d4 GIT binary patch literal 13017 zcmZ|VWmH>TwiR{`95@&9`pb8zPIVY z-^P$`p(+4HSWq-9^gllc0Fd>|mr!Y- zE2+J_e9c4zL9t`K~ThaNqu)uNE94q%W_asthpBiI=x`agsDO zGywv+fqpIj99C-ygzBvYsQrBVoc~Vrals!Vz#sT`B9AqS;h-x$hrbgA0{{F_{X3JI zrHQE>U=utOY$XUN*zA819IU3Vroas#7O8jnZy@*m0lC5cf2TPH{s0e0Nb`S}4Sp?G zbPxmp_)W3{GJte~zzYDB1YA&1Xi#9N0Eo9lK5F0LyQT5V!cR!ICyU6Dhi|Em!OiBC z=fzRxtOz(i?2I#! z3b>$!>ouJZXKiI%hXi4s+!0g@l*`~f)@e!{VkV#IXVHi!=xICe-`8v;3iR+EqP(MB zI^#0&)k$W%`BEJ{P0O;{!LJZS(^`KxacSN`=Ns%cO@ygjvuZo|(!^|jG-pbjp@FyG z!LnO`&t@pZ@M>-;2SXXfA(;O@5bB2lEg@{9CY1TZaI&K(BjQf%-I1(sSM62rk$VQ? zubvy5N)^}vBF8IYsjm1h4C9tK)zvL4~83MGEk4dEv!$S5RTl1z#ZNX%9a!-k1p>VA#0dly*;m<}iQ#40OCz8M5W(>TZ8G*k(K_lPI54EWPsOy*sd% zszJ;3ddt$Zwc_+X5JeNASTrJ9sW;gYQ(Mgkk`47|Z;mQ6FYKF%z8k~ygBb$+~p|3(HI9!4+Hy1~4uY5=s@ zirf5%9d6?;Lg)v>p|uUAtsPgOca%qe&3`KgGh@TR#n__iBpB0lJkjy;KDE^b!@ zw%Nt0j*8|s!a_T@*hO+_Cj{j@xQDLzB3JG*l_2qJ@(de`_Q`~+W z<%@&Ev-5Gs7vKDV=FBCmo4M4OgiSh-I{I3Ol!Z|x`m<%rck_+8w%kSR(dxKWtphZ1 znP4-XI>#g*?w8`&HGgs6;pk+PP zx)1Y;A;7O#+9-P}y3rQDP~kS(aWM$$)R}Kwg}*j9lBj0QIl- z+>c_|pW#;9Vq$)5Ff#)T6cjm3x#|yOwuUAz63BZ2|Es(q{7-rNTjo&ymcF+VhzS-C zED>1pH-%n608+pdKY#}94*(Z{tHE>)Wou=REeK%0#(%cjO2cMRM4%x;%S<^JmdpSB z`-R04iQo%nl+HcE)T)oWhAmN^5fSzN18nswT}~GPQO^NQ&PU7 zQU5ssVAu4|u2M!cuOPk`WiY9c-6j7BjkwrI%Hq)ajrON@3t!#yA#~_SBC_6Z7bqU8 z(PXxXSGhY*SAl8c{3sFe@Z_JrL4k3>!Wz8$KT8aA9XQXr64_ftXE-~z_~Qi*4l(F4 zIy?U`YQe|dJDc$@hez`@BI(+d56r%5@hDaMjgWcJ^-Xx{6jw~!c7$s_$FeR-aC^Le z?Z#i%t%uN=qB&c*CzGP!1Tmggp@0%Cd}4ltimPPG4#Kf+I#9)n5P1lN-I2rw2tFpk zW8MMsW|HCf%08cr7G)Xdd`;3c`*pPYWik}!BgFcjZ&h2T_@J=XIp*oaDwx?VR~}bn zya2esd1W_&#o*`~yIYNYdvqK{wa6{qn7N=&BQ!fb(G7G4&M2I#q5+EExFq}&jo9)0 zv-@68{hc8tqH4Cu+uv_v=Ny&n>~2=6*?t7FrlC$RS*8q!(i*wnOL3ajWQ#Eoey!?d zuq$JB!m9UwyYnzbUi_0*C+{cmXK?O&pYC2Z&w(V4{R3OrTX!DB%J2GZE?n zqj0Sn_l7+77S=`ocEkk17fV<3JC|mfjLvuy2=1+3K-yu5;5z=M8??reR34!r!`-`1YTlB?e2j0#US0cuXRG4am z%8*BfY8$yd^W$U!RHQiDgww>Y7mP}>6W^)b0z#r~ooYzMD*1(}dwT2@*5QxMpC_zE zgP%d+NREj&DzwXFfhb68%YYYR-2I}U`II2PyoNlFkT{R3EjoQ6JS0lo~)Y$}-;{F5Nk;J_<@RRyaBRv)YhSj#ts z=0N}xz!VPv^}h)sa6lml;6y7#$s(aN4`srt0i>W?5@ZgHs#N0}5H15{dbz9z8=5YQ z-S@;;zJ!m^aR?0&SuMHOGCI#*kwRC5k#$0yO!}$&o^;4SqE8_61ZXXe(|$Ka2?$KS zqj#u9dWDcUDAc9*9$ZbWfSV~HSZsv!{Z}<(5L-$%<227g}sMJr*;cjdlQf6 zfEy1#?N75|OU!palEL=Dhg0b+a-(fn)b&XKMVW5VTR6Jo1M;6<3~A}eWe$_+GJS7C zkrh}zx;*=)l$O$^$M-F8NcszE8Ber5bY@%B`9cBVTb8F^h8=A6l5R;$4WD+C?Z zvyCOwg0on1F-(UOY@V2{<7v7Dx&RM$@|SA7-q5zseJ$H_!@*HHGh0i>GJ65XRmT!t$27HW*=|owJJ{L%j3Yh{iIHJ*x!T9 zvh4a3kT7V4ZSEM=-L)hJBvd4=zgUSCnexYsBGd`UB4@Eq0Z3-4bw# zBEy!B;FE;)j5c<7Ky3#1;uFGt<7W5->2MKRs5c%WKNju6K$QbsOaaVGF^c`um2S9p#_-oIxcxWKrGnrVAH_< z0Gki?=bJ)xAV49&f#ctrq5hUC@acLt>H4sbfr2r*qdMuOlo8ju&9Ie!l3jYwjFX?w zLK-Lu!X@y&UaRrCY2AKM)spE6(jW1qHB;y4qLZe;87k5`g|DtY0{T@ITNs`%1r}Oh%e&Q; z;8yRb7e4xvpgtC?`d7o0I>1^em7Iv}Pb5smL}7T#t5&ST6-4miez6@AX>5&)?7NVBzrXfJG1aj8}rSdPbNajY&Wrw53BwSk4&+XJz=dIQc&B7NBd zw+A?_Y;K|6itPEI)vI%EQ1GGYy@BX?z-qEQC)FGt`7{q++YL!xVEyqTvj(riZ#*Vt z`I-5B-a4zfFp_|bQb@(KJKkvZ4L$LBWCToXaqqgKO( zJ`dZ(39UvM4;IdPJLfT-0KnOQ)`$vl9BaWgfo%ud19lMX*qcJfAiy91jSqnH-_r2U z8Uc5*eOLMmzUj(X;`);{v?*Fab^QeNZp!ofBovdocoJP&REr7XVvGo5>f84b_Dc4TDz2-M*EjJNX}X8jy6nMNhxNn(?@=rlAB!$-GsV1 zuY?u2V-5G{9YtU|WMIN0&t?V1m6Gu-qj;ui#6l7~)Rz+bFnoo`k^D7Lx6;^DVoEX7 za3I_Qa~SAe&WZ2bN;roVM$|dBTK>SS!x^t)1TC;>hMaj%9S*mL�&$I0%`N@P{x z9_2xyTPs$r3bR)oUf+DC^wDa7)U%mteRGskH^Hr`Jk6v%CI)}&iZ*F^LxB3JPTILo zd*H%_%^4!Fqx?5=XlJBuG`4n!vO;{gd!aNktvr5B4%LAoEw1wdnu~4`^vDxxr;J4C z;`Ef$B#s!}GZu%4vj#T^$;Z*iHjSBRJ~WaB|M(k>O&S z4e#gX#mR_#w)Rw-qxxMu(LRrHgrYQt_$-`;Lk?+tTuzO&S-xOW~0S*ByOm7PZrU54#c@ECgr}@~x0kLHtJma;b zoN=Ay+kTu#E9G}&aUM56@abbdqPJ2bVk*xAf<-74y*XrcLlGiy#2bE7TX)jHtzTW| z|GcW4(FPF@xj3wR@~-bZ8Sd{=lUG{Tl4RAn;pkjbBG5ZIZ^gpG$`)}9`xGGqaZzT^ z^!nmck`bSs+{MCR#(u|c|1CFGX1yXJqzV13n>;f&nXsBlc1i2;Snq78>05E#Y^h}g zX-)P+NpQ!w?AQz_?O3|n30K_GVUbBzpB!diLe1$oR_NO}(&qd%Nknp1+xfWb;AD(1 zKPdV!u*|CKn{e!JTsu>EydCp&YHsV5h_#6(6NP$?YDVEx%|&LqrBwv#2olH_}{l;?$ zeOm#NNE%$zaT-g811~$XMkp~X72*7^8=Tr>xEO*E^~cH)gX3-dM~;zY%I^7#0q&zUZ)(Z- zON5YG*=IMml4dd3kbJu$;fGXHZYwhW-2;^+jq4V4g=abZQRL}^!We1BccqD8?>t;| z9Va6(jnL*|7=LKkuPHwX^C(iPe&TDhUK_Q#6rYRmGI%7f39VI%c7iPCdrz)D!k!*f zs9|ULeKn@yd&}LI?Xs?kIr}lz^2Vau=-4$2`rGN-MY{OR@W*+#v2G;W8CsowXtrBQ z%=mVK%i|UuPY;yJcP`FTRC|EwB1DTubw2_3Pt0=|?!2%1+EZg=^5Q`B5(oUHnf7Hs zh6pTziguHMf_S^mG~sA6FOq_WbO5h3Z9c1vFBa``PGr!mkENPNdnUWxW>SF3fK)5) z$7aUUnJa67@9hiNp=`f;t-i6d9z9zL{&Ox|fOGu@>?7FM{|JNv3lA3gO`%Z`5CIU& z{qJR0DDK}C66ItMS7620dNz2Fgl1!!PobK(FN?%dS1C30sF>7<404e8v91A;NR^oH zGbmXPs;(0{{QdZ!o4%&-54IEUHtAcu1;cFd15n_-spP**V2CpPE&OO46dsL7mB@yr z6Vh{`x9rpic?I$%I^8r$mCm;ObqjyUr3glH(!Kdx&>r{-s(&4Myl0t8ZUG3jigWB6)P3s1b{+fX{I zI;lIvOUbSF8fM!T%uP1zs!DIPkjnpK2q&^0{uJ?QH0_Qr_33$mt}*(oB#t}PXlmO4 zb$zgBI5I7mJoe}u(S(tt`#kg@Aws2AQkyQY9DNu>PPL;tl~c_#)I)t#5!oTwiY%e> ztddNeF?4Cse0p6r{ew9Iq1qsdr%#8CQw{_1)Gr`$<3d8r%cJz?A)FFXj`G59c^1j9 zn$-bGLD*DBGIZUzY;oC3&3pzWld$zE#uzls9mG_vq6)Xtn}a6J_LL0JmkU@Uga?lN z?I4+$`lztNT++$K)U5N;!a-qHYw0S+%NSVFL5V5z~f2TXARF#fx6a`3t>c4JckV4lr!+RSM(a;qPbG~Wvz8rP-EW!5$Y9}IVUWxwB7 zsi9Uzct>0~KdtwRFZD+2_Xck7H6~NiyQ`MnNhs z@gZJ~ajR>Rg`@c2dpRNfZ?Caky`JLJH7|S4+{cFVvjPDcmOV+y>SUscHpulGK zBokE|NkpCqJ}f?%MFif6Ck7KS1((I27YZU3F9?8gdW2MDcP)HYW~9k?ll53Ws4$2O zanPsqQ@UY-z%vjW+f^ z6g7REPVLXTI8WE>?m1rt$YfgV3yUnQ@aZh^r_+6>kQ z!{D?U#f#5D(c##Dr0An_J?wec0~Yr1Dv2gxe;m#juKXHhPI4GA_DTvl_kkBdZYnC{ zfZ~UtRyYdDUmbO6a>E&j^!&$<4Z1y^C2G_cpzV9fBCQ62ghU!Ib=PV-)x0j){?F`s zNnS zXNfc>l171(`#34-ScY0Uymc_IQ03$87ewFZXY%gH04<}E@=E6@h}s-mM`2d3I&kuP za-xyaGemP&(bn&)De1@Wl-^u?#0<6si@~$>mxpEcTP}TKvn_5zBD(zpP$8T}J<;5s z$Ts%sYEr-GaP5m1vZjsOto`Qr4fs{4P zSLn?sqnR!#T)xZeLL7hSl0S6uOcUX7*S-qpx? zv1C;5Yq{3N$r{z9DWhgBGwM#;)ZG_B<|K&{GX?E2@K}9;iInlHi8(VRN zYtS0)s!nBCEGnji?FuKc`iXN>B#J@9$Lf_}5p`vkD5#nVwdz#?@NVXxZUE>84!Sp3 zf3U$|Bf!RhO?*@676gm|RPh1u-;;<$03g1i`Yux_d1}C_NCvB<)R_C6 zM_hCSrN85t?|#)NI`{)-rUP*X$`;?J_)sR1CrND>>V&>UVH?*^W%}6mT^S9QS+zr0 zSvo)2_!tEQPu}{}JI>-Kf4k9jmlbm-kf&O|iakHXV1F+FOKIg4wC4V-glI48VzU~^i$zx-B0YmHd zy+ecel~8bbrcBW;hUSS2PjEtu-{OlRK8K;?!_|YjolLSKL1t*cpixN^)JcUs^)@#= zR=i`%FIIV-l8wZW;wYOv!M3b!jO^nkW@}n#cjj9^FX-2g^P6I`wlkh2|ZkSA*2OqF$7KNE1Z|BS+sv%b>B%!&8h6#Xy>iKO)$)Ri6RHG#T`%L|q|ZKa1GrjlT?v{Zbp8J68&biE zkOejmY%$n!ur+VKwT)GPORj(S;1K-Yst5vSQL1>@9%S{6*(66qQWVzZX%S`&&nMzG zp_s>RJVwuSlXv(&z^}A1nXWl@mk8NH`h6CRK`aMfo}hN~b4Nm#<8RD8kvtpk_7N7g zEsS18;j9~uRxRryq$VdU&QTFsoc@?-C**pvXToh!%JUct+2JPJofi7by$pxz%| znpH-7s{9t)=!yKmTB@--vno$ZGlm^aDI_r+dX;F99cv~swO>k2y6b~}3%Q8Wu*wKI z?=FMjnlWm1Kkj(D_eofkWDWk+zRrD|D-N@^Y{`Ms>p!qGIOwktWr5Xi^- zavIpgxJ{*}8B7ZcD_6PGG0p&< zff6>MPNKw1v*?Yu1L4z*h4NHIZwbDh6QXfd{FI>%dudtGT3{q<48i7J`gb*jf5P7g z4sRRSZmXp z>LCq61Z~JA=Sui>;$9_R99c<~h43v0tu)1h-hw!LohJ5>=~rvx3q)c15sBunp-`E4 zcS%+&h6ZD8!)ae*_>{UYE|YCNLw>L^Qk9Yd4~-b<-KtQ$I6+EBd2UzSimfeyoUxeS z;;DcQdgK(;^uTQ6vD_MaamZec#0k4_Dr>tF$zxGCd#hV6HCd}Pyc>n008r(sc4_PJLDzNV=y;WylN9>M6t_ouSF-H522C;< zy1=!CH>S;Aa{@Rd`)d8y!sO~n;J}iIzoa&6XL5{yyWaY{# zGkm{>C3s5V58w`GtqnmcZm|&l%TKj{f+HRD3g{0?tenvLL9C9JfhCXIuUQC2YM8-i zlm0_Y92eP%1%pKS)AfXT0ii0-A;GcpMH-kPG&SAo+L70l0IqSrx8`yMoC2F*cflTk zJqLRY_Tf#T8xZgsp!n(ERLQrX5P0oJu|fC=IXx&Vma)!9q93zEKbv1EXvvk;E_XLC zRZ#P%e41~@NFb&!a(h@4S$54Ld)};sUR-RTmxvwy_dTlgt<7^5K+Mnf@$}qrGM@GV zQmx@BcL&Mua2qKYh3=e#>h@!G*e^aCf6awEevZ!VAS(P4~g12@$Z9 zb<#t>K&T3n)&tFf8Oa5dpp=bXjQNSWh+P&scP}cxtg%Fl3=f4(C*7#ULKwTCWD@Lz zj)N_j{<@_ZEM5?|LW0is!uIIHB=1epbAP>~nYa{1(ijp0uQO_;>^9>DbDH_OtS+fhYkQ5+PD*ycoMhGxjCS)oSctbAEfY7P>^ZHR$32O93cWmN+zTzpuoHCa zlhr|oraUTEQsb@`S+mC#^baSWt&!J1x$T8ET6mg`2xIXeudDz^*JYe0@vbsA5L+-! zg4GdX#zrz&+_b%Ri{ka{cH(O2XC={C((-MLR0B=fdMT zW$on3si?fylR$9^h*=j=vV)?ph76r{9pJLhswc8}-4o#LoVA%9`F?w4$^=YOjkuRb zme?|@zyACl6*ss9M!BsjOPv~8xkSrl$wa;RMKJh`q$w#!A^h&R>T3}Z>y{yppee{C zGjc7w{QyZ*gS~i)3Lajdde)U}wDO&+L0n%V8-d_=JwiW=A6on|tnw|F-qC5%yc;I+ z@KaoC>z2vhpt3DuY`bj5>~Nxe?eGV#o($xJTceYtxK}Qj&|e=IWaar2&)*SQ2RBKk zsEXbx&Ut^fE};=1>kj;<_keg0&Hz%dRA4`VWd_Ru_T!tvP#_2{fTF{&4WI9bU$ zhk|_uls@G?O*Uuye0Ss3gHzgo!PBmlVDpZuMWp8&owA%o_*$Zn4h3JSGW}KA!mo&I zD#uOM#W|W?iZTA!p&lQZlA)GFy5jo7`{X}pmOAdZRD_n|`n-#LTM%Zt0!xD@k8!jF zaAhCs>%Tsyoi&U}v7zTwdxcS#%BDu-D!F^p|5*M-P)1Jle39J5l4bQtvp*bttE%F3 zx#Usd1KZDKX4GA~No%WGeg$dDzB+f*#30#lPm|W_#1+eJ_#~lWDtQxn>EPu$-%T0{ zPQO6*lI5Kit&KmD22F{|{hgc84*vmgT-!bI_VLH1vY*uQi!A)5_N{LD)Jgv6Pjri8{5 zihP5bZLkw#*MPITU7u0%UCOLp9nC(yo`)9)<{b9~Kw2?gdP%HcAH)St=ENq~G?!W{ z7%nib4Y!KTt73#A^T0@M2N0Qi4`Fi`&rhhj_Ia{Nx21yIKVOrj@iT@ev&$GD9r`3y za42nRXbCE)j#>|q%|I#f-5i@dd`4aBy*^#9*!3-x*8%OMH($V=$Np$>^C5wd=HD6O zIBH`QbZT2yU^gm==3g6E{;|zCBJ0TJ776f)dh1;v1i)z_3RViN99U(r8enzb{ChV5 zLIZHg^p-xD2GIGrw(au1(VD#wW-U7yR5B^7A2a`0=W>AdrKk4}(CAH#7eBBocEkTl zZNx)1>N-#`a;nOEo=2d*cU2if?szYf&ThH)Gv6?sm}IQR?m{ap>}TKkGjweIq3 zeo7jekd4vtk9O?}R299xV8PG?*UCdl|DSg;h5onaP+uqEeEvlBuOAz_aSwTM*%wtT zI96djSP+U((s5Qn$T9q=)d?7Zlt-FMjd$#@*~7u#O}?i2`_~MqaCNQ9(+5Y(kUJRQ z5{u{HG-2e59A5K9@_1y^b$;7e@=qC?vAMG$0?{lD_h}#t+pJ2ToRwU6Jxo2cbJa>f3XIKIY$oezu&j0wcfXQ|x9iCV!)s?57S3MSrhcY%Yf z8`BxEAX(X|`%@-68VN!94YpCv+b(^lqZ z=GcL*rtYoh?VCvrb_*%Iv{l0F{K}9@fB>whfQCIYV5zMM6UEK$Y1_8f6SQ!Uplsnh zi6jzs6DIThj<$CSiw=Y9JZyT|Qzx^(xPz@9S3z8NzZJdPz4&RG^xKRhH2Ap4K_ z29(LeHRH>i@r&#SC9Ys4X@h4 zC`fKH)bju{2D)`v#iuw7L$y+!$*kEeKs)fc0dQ)D`X%`dkwMxD7x`013X;Gm<>Spj z{&P+kfpgs)tTk8%u&!Xe-u$~M1mOX={9l_w8gF+2P_??Opi3(YqIzO(nk_2=S=gYD9;5*T_Pstsm{$qsYwl9UjRAK z;EHQnX_$h917qF%Da=x_VTCWL?RP*edLZ0Kgh&aJqT<#*l4f`U{kd_52!V<_GMHuY@8(H@vRr5zz6 z-6>kY#yK(6`m^W5xCji~e5X8ExLELl&+PDbS>}HR?6kC8f9;Fx$K-i(TgV~Wkn$r| zkl6l>GdiHtc%m?sF}3v)SyugMn&X#`GkWTSHN3j(uguMmWO*u{RPGC+-eU7!hJike<2BC0Vqt!220x;{C*N&sJw+Ez@hvH@uok9t=q1hpsh_78a3D*&IoJ>)$%#<{FGVe$Q5Hf 0xFF: + return wrapErrorf(ErrTooManyFrames, "lacing %d frames", nFrames) + } + size := []byte{byte(nFrames - 1)} + for i := 0; i < nFrames-1; i++ { + n := len(b[i]) + for ; n > 0xFF; n -= 0xFF { + size = append(size, 0xFF) + } + size = append(size, byte(n)) + } + if _, err := l.w.Write(size); err != nil { + return err + } + for i := 0; i < nFrames; i++ { + if _, err := l.w.Write(b[i]); err != nil { + return err + } + } + return nil +} + +func (l *fixedLacer) Write(b [][]byte) error { + nFrames := len(b) + switch { + case nFrames == 0: + return nil + case nFrames > 0xFF: + return wrapErrorf(ErrTooManyFrames, "lacing %d frames", nFrames) + } + for i := 1; i < nFrames; i++ { + if len(b[i]) != len(b[0]) { + return wrapErrorf( + ErrUnevenFixedLace, "lacing %d bytes on %d bytes frame", len(b[i]), len(b[0]), + ) + } + } + if _, err := l.w.Write([]byte{byte(nFrames - 1)}); err != nil { + return err + } + for i := 0; i < nFrames; i++ { + if _, err := l.w.Write(b[i]); err != nil { + return err + } + } + return nil +} + +func (l *ebmlLacer) Write(b [][]byte) error { + nFrames := len(b) + switch { + case nFrames == 0: + return nil + case nFrames > 0xFF: + return wrapErrorf(ErrTooManyFrames, "lacing %d frames", nFrames) + } + size := []byte{byte(nFrames - 1)} + + n, err := encodeElementID(uint64(len(b[0]))) + if err != nil { + return err + } + size = append(size, n...) + + frameSizePrev := int64(len(b[0])) + for i := 1; i < nFrames-1; i++ { + frameSize := int64(len(b[i])) + n, err := encodeVInt(frameSize - frameSizePrev) + frameSizePrev = frameSize + if err != nil { + return err + } + size = append(size, n...) + } + if _, err := l.w.Write(size); err != nil { + return err + } + for i := 0; i < nFrames; i++ { + if _, err := l.w.Write(b[i]); err != nil { + return err + } + } + return nil +} + +// NewNoLacer creates pass-through Lacer for not laced data. +func NewNoLacer(w io.Writer) Lacer { + return &noLacer{w} +} + +// NewXiphLacer creates Lacer for Xiph laced data. +func NewXiphLacer(w io.Writer) Lacer { + return &xiphLacer{w} +} + +// NewFixedLacer creates Lacer for Fixed laced data. +func NewFixedLacer(w io.Writer) Lacer { + return &fixedLacer{w} +} + +// NewEBMLLacer creates Lacer for EBML laced data. +func NewEBMLLacer(w io.Writer) Lacer { + return &ebmlLacer{w} +} diff --git a/pkg/ebml-go/lacer_test.go b/pkg/ebml-go/lacer_test.go new file mode 100644 index 0000000..b07f1a1 --- /dev/null +++ b/pkg/ebml-go/lacer_test.go @@ -0,0 +1,187 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ebml + +import ( + "bytes" + "io" + "testing" + + "github.com/at-wat/ebml-go/internal/errs" +) + +func TestLacer(t *testing.T) { + cases := map[string]struct { + newLacer func(io.Writer) Lacer + frames [][]byte + b []byte + err error + }{ + "NoLaceEmpty": { + newLacer: NewNoLacer, + frames: [][]byte{}, + b: []byte{}, + err: nil, + }, + "NoLaceTooMany": { + newLacer: NewNoLacer, + frames: make([][]byte, 2), + b: []byte{}, + err: ErrTooManyFrames, + }, + "Xiph": { + newLacer: NewXiphLacer, + frames: [][]byte{ + bytes.Repeat([]byte{0xAA}, 256), + bytes.Repeat([]byte{0xCC}, 16), + bytes.Repeat([]byte{0x55}, 8), + }, + b: append( + []byte{ + 0x02, + 0xFF, 0x01, // 256 bytes + 0x10, // 16 bytes + }, + bytes.Join( + [][]byte{ + bytes.Repeat([]byte{0xAA}, 256), + bytes.Repeat([]byte{0xCC}, 16), + bytes.Repeat([]byte{0x55}, 8)}, []byte{})...), + err: nil, + }, + "XiphEmpty": { + newLacer: NewXiphLacer, + frames: [][]byte{}, + b: []byte{}, + err: nil, + }, + "XiphTooLong": { + newLacer: NewXiphLacer, + frames: make([][]byte, 256), + b: nil, + err: ErrTooManyFrames, + }, + "Fixed": { + newLacer: NewFixedLacer, + frames: [][]byte{ + bytes.Repeat([]byte{0xAA}, 16), + bytes.Repeat([]byte{0xCC}, 16), + bytes.Repeat([]byte{0x55}, 16), + }, + b: append( + []byte{ + 0x02, + }, + bytes.Join( + [][]byte{ + bytes.Repeat([]byte{0xAA}, 16), + bytes.Repeat([]byte{0xCC}, 16), + bytes.Repeat([]byte{0x55}, 16), + }, []byte{})...), + err: nil, + }, + "FixedEmpty": { + newLacer: NewFixedLacer, + frames: [][]byte{}, + b: []byte{}, + err: nil, + }, + "FixedUneven": { + newLacer: NewFixedLacer, + frames: [][]byte{ + bytes.Repeat([]byte{0xAA}, 16), + bytes.Repeat([]byte{0xCC}, 16), + bytes.Repeat([]byte{0x55}, 15), + }, + b: nil, + err: ErrUnevenFixedLace, + }, + "FixedTooLong": { + newLacer: NewFixedLacer, + frames: make([][]byte, 256), + b: nil, + err: ErrTooManyFrames, + }, + "EBML": { + newLacer: NewEBMLLacer, + frames: [][]byte{ + bytes.Repeat([]byte{0xAA}, 800), + bytes.Repeat([]byte{0xCC}, 500), + bytes.Repeat([]byte{0x55}, 100), + }, + b: append( + []byte{ + 0x02, + 0x43, 0x20, // 800 bytes + 0x5E, 0xD3, // 500 bytes + }, + bytes.Join( + [][]byte{ + bytes.Repeat([]byte{0xAA}, 800), + bytes.Repeat([]byte{0xCC}, 500), + bytes.Repeat([]byte{0x55}, 100), + }, []byte{})...), + err: nil, + }, + "EBMLEmpty": { + newLacer: NewEBMLLacer, + frames: [][]byte{}, + b: []byte{}, + err: nil, + }, + "EBMLTooLong": { + newLacer: NewEBMLLacer, + frames: make([][]byte, 256), + b: nil, + err: ErrTooManyFrames, + }, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + var buf bytes.Buffer + l := c.newLacer(&buf) + err := l.Write(c.frames) + if !errs.Is(err, c.err) { + t.Fatalf("Expected error: '%v', got: '%v'", c.err, err) + } + if !bytes.Equal(c.b, buf.Bytes()) { + t.Errorf("Expected data: %v, \n got: %v", c.b, buf.Bytes()) + } + }) + } +} + +func TestLacer_WriterError(t *testing.T) { + lacers := map[string]struct { + frames [][]byte + newLacer func(io.Writer) Lacer + n int + }{ + "NoLacer": {[][]byte{{0x01, 0x02}}, NewNoLacer, 3}, + "XiphLacer": {[][]byte{{0x01}, {0x02}}, NewXiphLacer, 4}, + "FixedLacer": {[][]byte{{0x01}, {0x02}}, NewFixedLacer, 3}, + "EBMLLacer": {[][]byte{{0x01}, {0x02}}, NewEBMLLacer, 4}, + } + for name, c := range lacers { + t.Run(name, func(t *testing.T) { + for l := 0; l < c.n-1; l++ { + lacer := c.newLacer(&limitedDummyWriter{limit: l}) + if err := lacer.Write(c.frames); !errs.Is(err, bytes.ErrTooLarge) { + t.Errorf("Expected error against too large data (Writer size limit: %d): '%v', got '%v'", l, bytes.ErrTooLarge, err) + } + } + }) + } +} diff --git a/pkg/ebml-go/marshal.go b/pkg/ebml-go/marshal.go new file mode 100644 index 0000000..6c54c8f --- /dev/null +++ b/pkg/ebml-go/marshal.go @@ -0,0 +1,328 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ebml + +import ( + "bytes" + "errors" + "io" + "reflect" +) + +// ErrUnsupportedElement means that a element name is known but unsupported in this version of ebml-go. +var ErrUnsupportedElement = errors.New("unsupported element") + +// ErrNonStringMapKey is returned if input is map and key is not a string. +var ErrNonStringMapKey = errors.New("non-string map key") + +// Marshal struct to EBML bytes. +// +// Examples of struct field tags: +// +// // Field appears as element "EBMLVersion". +// Field uint64 `ebml:EBMLVersion` +// +// // Field appears as element "EBMLVersion" and +// // the field is omitted from the output if the value is empty. +// Field uint64 `ebml:TheElement,omitempty` +// +// // Field appears as element "EBMLVersion" and +// // the field is omitted from the output if the value is empty. +// EBMLVersion uint64 `ebml:,omitempty` +// +// // Field appears as master element "Segment" and +// // the size of the element contents is left unknown for streaming data. +// Field struct{} `ebml:Segment,size=unknown` +// +// // Field appears as master element "Segment" and +// // the size of the element contents is left unknown for streaming data. +// // This style may be deprecated in the future. +// Field struct{} `ebml:Segment,inf` +// +// // Field appears as element "EBMLVersion" and +// // the size of the element data is reserved by 4 bytes. +// Field uint64 `ebml:EBMLVersion,size=4` +func Marshal(val interface{}, w io.Writer, opts ...MarshalOption) error { + options := &MarshalOptions{} + for _, o := range opts { + if err := o(options); err != nil { + return err + } + } + vo := reflect.ValueOf(val) + if vo.Kind() != reflect.Ptr { + return wrapErrorf(ErrInvalidType, "marshalling to %T", val) + } + + _, err := marshalImpl(vo.Elem(), w, 0, nil, options) + return err +} + +func pealElem(v reflect.Value, binary, omitEmpty bool) ([]reflect.Value, bool) { + for { + switch v.Kind() { + case reflect.Interface, reflect.Ptr: + if v.IsNil() { + return nil, false + } + v = v.Elem() + case reflect.Slice: + if binary { + if omitEmpty && v.Len() == 0 { + return nil, false + } + return []reflect.Value{v}, true + } + var lst []reflect.Value + l := v.Len() + for i := 0; i < l; i++ { + vv, ok := pealElem(v.Index(i), false, omitEmpty) + if !ok { + continue + } + lst = append(lst, vv...) + } + return lst, true + case reflect.Chan: + return []reflect.Value{v}, true + default: + if omitEmpty && deepIsZero(v) { + return nil, false + } + return []reflect.Value{v}, true + } + } +} + +func deepIsZero(v reflect.Value) bool { + return reflect.DeepEqual(reflect.Zero(v.Type()).Interface(), v.Interface()) +} + +func marshalImpl(vo reflect.Value, w io.Writer, pos uint64, parent *Element, options *MarshalOptions) (uint64, error) { + var l int + var tagFieldFunc func(int) (*structTag, reflect.Value, error) + + switch vo.Kind() { + case reflect.Struct: + l = vo.NumField() + tagFieldFunc = func(i int) (*structTag, reflect.Value, error) { + tag := &structTag{} + tn := vo.Type().Field(i) + if n, ok := tn.Tag.Lookup("ebml"); ok { + var err error + if tag, err = parseTag(n); err != nil { + return nil, reflect.Value{}, err + } + } + if tag.name == "" { + tag.name = tn.Name + } + return tag, vo.Field(i), nil + } + case reflect.Map: + l = vo.Len() + keys := vo.MapKeys() + tagFieldFunc = func(i int) (*structTag, reflect.Value, error) { + name := keys[i] + if name.Kind() != reflect.String { + return nil, reflect.Value{}, ErrNonStringMapKey + } + return &structTag{name: name.String()}, vo.MapIndex(name), nil + } + default: + return pos, ErrIncompatibleType + } + + for i := 0; i < l; i++ { + tag, vn, err := tagFieldFunc(i) + if err != nil { + return pos, err + } + + t, err := ElementTypeFromString(tag.name) + if err != nil { + return pos, err + } + e, ok := table[t] + if !ok { + return pos, wrapErrorf(ErrUnsupportedElement, "marshalling \"%s\"", t) + } + + unknown := tag.size == SizeUnknown + + lst, ok := pealElem(vn, e.t == DataTypeBinary, tag.omitEmpty) + if !ok { + continue + } + + writeOne := func(vn reflect.Value) (uint64, error) { + // Write element ID + var headerSize uint64 + n, err := w.Write(e.b) + if err != nil { + return pos, err + } + headerSize += uint64(n) + + var bw io.Writer + if unknown { + // Directly write length unspecified element + bsz := encodeDataSize(uint64(SizeUnknown), 0) + n, err := w.Write(bsz) + if err != nil { + return pos, err + } + headerSize += uint64(n) + bw = w + } else { + bw = &bytes.Buffer{} + } + + var elem *Element + if len(options.hooks) > 0 { + elem = &Element{ + Value: vn.Interface(), + Name: tag.name, + Type: t, + Position: pos, + Size: SizeUnknown, + Parent: parent, + } + } + + var size uint64 + if e.t == DataTypeMaster { + p, err := marshalImpl(vn, bw, pos+headerSize, elem, options) + if err != nil { + return pos, err + } + size = p - pos - headerSize + } else { + bc, err := perTypeEncoder[e.t](vn.Interface(), tag.size) + if err != nil { + return pos, err + } + n, err := bw.Write(bc) + if err != nil { + return pos, err + } + size = uint64(n) + } + + // Write element with length + if !unknown { + if len(options.hooks) > 0 { + elem.Size = size + } + bsz := encodeDataSize(size, options.dataSizeLen) + n, err := w.Write(bsz) + if err != nil { + return pos, err + } + headerSize += uint64(n) + + if _, err := w.Write(bw.(*bytes.Buffer).Bytes()); err != nil { + return pos, err + } + } + for _, cb := range options.hooks { + cb(elem) + } + pos += headerSize + size + return pos, nil + } + + for _, vn := range lst { + var err error + switch vn.Kind() { + case reflect.Chan: + for { + val, ok := vn.Recv() + if !ok { + break + } + lst, ok := pealElem(val, e.t == DataTypeBinary, tag.omitEmpty) + if !ok { + return pos, wrapErrorf( + ErrIncompatibleType, "marshalling %s from channel", val.Type(), + ) + } + if len(lst) != 1 { + return pos, wrapErrorf( + ErrIncompatibleType, "marshalling %s from channel", val.Type(), + ) + } + pos, err = writeOne(lst[0]) + } + case reflect.Func: + ret := vn.Call(nil) + lenRet := len(ret) + if lenRet != 1 && lenRet != 2 { + return pos, wrapErrorf(ErrIncompatibleType, "number of return value must be 1 or 2 but %d", lenRet) + } + val := ret[0] + if lenRet == 2 { + errVal := ret[1] + if errVal.Type().String() != "error" { + return pos, wrapErrorf(ErrIncompatibleType, "2nd return value must be error but %s", errVal.Type()) + } + if iFace := errVal.Interface(); iFace != nil { + return pos, iFace.(error) + } + } + lst, ok := pealElem(val, e.t == DataTypeBinary, tag.omitEmpty) + if !ok { + return pos, wrapErrorf( + ErrIncompatibleType, "marshalling %s from func", val.Type(), + ) + } + for _, l := range lst { + pos, err = writeOne(l) + } + default: + pos, err = writeOne(vn) + } + if err != nil { + return pos, err + } + } + } + return pos, nil +} + +// MarshalOption configures a MarshalOptions struct. +type MarshalOption func(*MarshalOptions) error + +// MarshalOptions stores options for marshalling. +type MarshalOptions struct { + dataSizeLen uint64 + hooks []func(elem *Element) +} + +// WithDataSizeLen returns an MarshalOption which sets number of reserved bytes of element data size. +func WithDataSizeLen(l int) MarshalOption { + return func(opts *MarshalOptions) error { + opts.dataSizeLen = uint64(l) + return nil + } +} + +// WithElementWriteHooks returns an MarshalOption which registers element hooks. +func WithElementWriteHooks(hooks ...func(*Element)) MarshalOption { + return func(opts *MarshalOptions) error { + opts.hooks = hooks + return nil + } +} diff --git a/pkg/ebml-go/marshal_roundtrip_test.go b/pkg/ebml-go/marshal_roundtrip_test.go new file mode 100644 index 0000000..041b833 --- /dev/null +++ b/pkg/ebml-go/marshal_roundtrip_test.go @@ -0,0 +1,113 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ebml_test + +import ( + "bytes" + "reflect" + "testing" + "time" + + "github.com/at-wat/ebml-go" + "github.com/at-wat/ebml-go/webm" +) + +func TestMarshal_RoundtripWebM(t *testing.T) { + webm0 := struct { + Header webm.EBMLHeader `ebml:"EBML"` + Segment webm.Segment `ebml:"Segment,size=unknown"` + }{ + Header: webm.EBMLHeader{ + EBMLVersion: 1, + EBMLReadVersion: 1, + EBMLMaxIDLength: 4, + EBMLMaxSizeLength: 8, + DocType: "webm", + DocTypeVersion: 2, + DocTypeReadVersion: 2, + }, + Segment: webm.Segment{ + Info: webm.Info{ + TimecodeScale: 1000000, // 1ms + MuxingApp: "ebml-go example", + WritingApp: "ebml-go example", + DateUTC: time.Now().Truncate(time.Millisecond), + }, + Tracks: webm.Tracks{ + TrackEntry: []webm.TrackEntry{ + { + Name: "Video", + TrackNumber: 1, + TrackUID: 12345, + CodecID: "V_VP8", + TrackType: 1, + DefaultDuration: 33333333, + Video: &webm.Video{ + PixelWidth: 320, + PixelHeight: 240, + }, + CodecPrivate: []byte{0x01, 0x02}, + }, + { + Name: "Audio", + TrackNumber: 2, + TrackUID: 54321, + CodecID: "A_OPUS", + TrackType: 2, + DefaultDuration: 33333333, + Audio: &webm.Audio{ + SamplingFrequency: 48000.0, + Channels: 2, + }, + }, + }, + }, + Cluster: []webm.Cluster{ + { + Timecode: 0, + }, + { + Timecode: 1234567, + }, + }, + Cues: &webm.Cues{ + CuePoint: []webm.CuePoint{ + { + CueTime: 1, + CueTrackPositions: []webm.CueTrackPosition{ + {CueTrack: 2, CueClusterPosition: 3, CueBlockNumber: 4}, + }, + }, + }, + }, + }, + } + + var b bytes.Buffer + if err := ebml.Marshal(&webm0, &b); err != nil { + t.Fatalf("Failed to Marshal: '%v'", err) + } + var webm1 struct { + Header webm.EBMLHeader `ebml:"EBML"` + Segment webm.Segment `ebml:"Segment,size=unknown"` + } + if err := ebml.Unmarshal(bytes.NewBuffer(b.Bytes()), &webm1); err != nil { + t.Fatalf("Failed to Unmarshal: '%v'", err) + } + + if !reflect.DeepEqual(webm0, webm1) { + t.Errorf("Roundtrip result doesn't match original\nexpected: %+v\n got: %+v", webm0, webm1) + } +} diff --git a/pkg/ebml-go/marshal_test.go b/pkg/ebml-go/marshal_test.go new file mode 100644 index 0000000..44ecd7f --- /dev/null +++ b/pkg/ebml-go/marshal_test.go @@ -0,0 +1,715 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ebml + +import ( + "bytes" + "errors" + "fmt" + "reflect" + "testing" + + "github.com/at-wat/ebml-go/internal/errs" +) + +func TestMarshal(t *testing.T) { + type TestOmitempty struct { + DocType string `ebml:"EBMLDocType,omitempty"` + DocTypeVersion uint64 `ebml:"EBMLDocTypeVersion,omitempty"` + SeekID []byte `ebml:"SeekID,omitempty"` + } + type TestNoOmitempty struct { + DocType string `ebml:"EBMLDocType"` + DocTypeVersion uint64 `ebml:"EBMLDocTypeVersion"` + SeekID []byte `ebml:"SeekID"` + } + type TestSliceOmitempty struct { + DocTypeVersion []uint64 `ebml:"EBMLDocTypeVersion,omitempty"` + } + type TestSliceNoOmitempty struct { + DocTypeVersion []uint64 `ebml:"EBMLDocTypeVersion"` + } + type TestSized struct { + DocType string `ebml:"EBMLDocType,size=3"` + DocTypeVersion uint64 `ebml:"EBMLDocTypeVersion,size=2"` + Duration0 float32 `ebml:"Duration,size=8"` + Duration1 float64 `ebml:"Duration,size=4"` + SeekID []byte `ebml:"SeekID,size=2"` + } + type TestPtr struct { + DocType *string `ebml:"EBMLDocType"` + DocTypeVersion *uint64 `ebml:"EBMLDocTypeVersion"` + } + type TestPtrOmitempty struct { + DocType *string `ebml:"EBMLDocType,omitempty"` + DocTypeVersion *uint64 `ebml:"EBMLDocTypeVersion,omitempty"` + } + type TestInterface struct { + DocType interface{} `ebml:"EBMLDocType"` + DocTypeVersion interface{} `ebml:"EBMLDocTypeVersion"` + } + type TestBlocks struct { + Block Block `ebml:"SimpleBlock"` + } + + var str string + var uinteger uint64 + + testCases := map[string]struct { + input interface{} + expected [][]byte // one of + }{ + "Omitempty": { + &struct{ EBML TestOmitempty }{}, + [][]byte{{0x1a, 0x45, 0xDF, 0xA3, 0x80}}, + }, + "NoOmitempty": { + &struct{ EBML TestNoOmitempty }{}, + [][]byte{ + { + 0x1A, 0x45, 0xDF, 0xA3, 0x8A, + 0x42, 0x82, 0x80, + 0x42, 0x87, 0x81, 0x00, + 0x53, 0xAB, 0x80, + }, + }, + }, + "SliceOmitempty": { + &struct { + EBML TestSliceOmitempty + }{TestSliceOmitempty{make([]uint64, 0)}}, + [][]byte{{0x1a, 0x45, 0xDF, 0xA3, 0x80}}, + }, + "SliceOmitemptyNested": { + &struct { + EBML []TestSliceOmitempty `ebml:"EBML,omitempty"` + }{make([]TestSliceOmitempty, 3)}, + [][]byte{{}}, + }, + "SliceNoOmitempty": { + &struct { + EBML TestSliceNoOmitempty + }{TestSliceNoOmitempty{make([]uint64, 2)}}, + [][]byte{ + { + 0x1a, 0x45, 0xDF, 0xA3, 0x88, + 0x42, 0x87, 0x81, 0x00, + 0x42, 0x87, 0x81, 0x00, + }, + }, + }, + "Sized": { + &struct{ EBML TestSized }{TestSized{"a", 1, 0.0, 0.0, []byte{0x01}}}, + [][]byte{ + { + 0x1A, 0x45, 0xDF, 0xA3, 0xA2, + 0x42, 0x82, 0x83, 0x61, 0x00, 0x00, + 0x42, 0x87, 0x82, 0x00, 0x01, + 0x44, 0x89, 0x88, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x44, 0x89, 0x84, 0x00, 0x00, 0x00, 0x00, + 0x53, 0xAB, 0x82, 0x01, 0x00, + }, + }, + }, + "SizedAndOverflow": { + &struct{ EBML TestSized }{TestSized{"abc", 0x012345, 0.0, 0.0, []byte{0x01, 0x02, 0x03}}}, + [][]byte{ + { + 0x1A, 0x45, 0xDF, 0xA3, 0xA4, + 0x42, 0x82, 0x83, 0x61, 0x62, 0x63, + 0x42, 0x87, 0x83, 0x01, 0x23, 0x45, + 0x44, 0x89, 0x88, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x44, 0x89, 0x84, 0x00, 0x00, 0x00, 0x00, + 0x53, 0xAB, 0x83, 0x01, 0x02, 0x03, + }, + }, + }, + "Ptr": { + &struct{ EBML TestPtr }{TestPtr{&str, &uinteger}}, + [][]byte{ + { + 0x1A, 0x45, 0xDF, 0xA3, 0x87, + 0x42, 0x82, 0x80, + 0x42, 0x87, 0x81, 0x00, + }, + }, + }, + "PtrOmitempty": { + &struct{ EBML TestPtrOmitempty }{TestPtrOmitempty{&str, &uinteger}}, + [][]byte{{0x1A, 0x45, 0xDF, 0xA3, 0x80}}, + }, + "Interface": { + &struct{ EBML TestInterface }{TestInterface{str, uinteger}}, + [][]byte{ + { + 0x1A, 0x45, 0xDF, 0xA3, 0x87, + 0x42, 0x82, 0x80, + 0x42, 0x87, 0x81, 0x00, + }, + }, + }, + "InterfacePtr": { + &struct{ EBML TestInterface }{TestInterface{&str, &uinteger}}, + [][]byte{ + { + 0x1A, 0x45, 0xDF, 0xA3, 0x87, + 0x42, 0x82, 0x80, + 0x42, 0x87, 0x81, 0x00, + }, + }, + }, + "Map": { + &map[string]interface{}{ + "Info": map[string]interface{}{ + "MuxingApp": "test", + "WritingApp": "abcd", + }, + }, + [][]byte{ + { + 0x15, 0x49, 0xA9, 0x66, 0x8E, + 0x4D, 0x80, 0x84, 0x74, 0x65, 0x73, 0x74, + 0x57, 0x41, 0x84, 0x61, 0x62, 0x63, 0x64, + }, + { // Go map element order is unstable + 0x15, 0x49, 0xA9, 0x66, 0x90, + 0x57, 0x41, 0x85, 0x61, 0x62, 0x63, 0x64, 0x00, + 0x4D, 0x80, 0x85, 0x74, 0x65, 0x73, 0x74, 0x00, + }, + }, + }, + "Block": { + &TestBlocks{ + Block: Block{ + TrackNumber: 0x01, Timecode: 0x0123, Lacing: LacingNo, Data: [][]byte{{0x01}}, + }, + }, + [][]byte{{0xA3, 0x85, 0x81, 0x01, 0x23, 0x00, 0x01}}, + }, + "BlockXiph": { + &TestBlocks{ + Block: Block{ + TrackNumber: 0x01, Timecode: 0x0123, Lacing: LacingXiph, Data: [][]byte{{0x01}, {0x02}}, + }, + }, + [][]byte{{0xA3, 0x88, 0x81, 0x01, 0x23, 0x02, 0x01, 0x01, 0x01, 0x02}}, + }, + "BlockFixed": { + &TestBlocks{ + Block: Block{ + TrackNumber: 0x01, Timecode: 0x0123, Lacing: LacingFixed, Data: [][]byte{{0x01}, {0x02}}, + }, + }, + [][]byte{{0xA3, 0x87, 0x81, 0x01, 0x23, 0x04, 0x01, 0x01, 0x02}}, + }, + "BlockEBML": { + &TestBlocks{ + Block: Block{ + TrackNumber: 0x01, Timecode: 0x0123, Lacing: LacingEBML, Data: [][]byte{{0x01}, {0x02}}, + }, + }, + [][]byte{{0xA3, 0x88, 0x81, 0x01, 0x23, 0x06, 0x01, 0x81, 0x01, 0x02}}, + }, + } + + for n, c := range testCases { + t.Run(n, func(t *testing.T) { + var b bytes.Buffer + if err := Marshal(c.input, &b); err != nil { + t.Fatalf("Unexpected error: '%v'", err) + } + for _, expected := range c.expected { + if bytes.Equal(expected, b.Bytes()) { + return + } + } + t.Errorf("Marshaled binary doesn't match:\n expected one of:\n%v,\ngot:\n%v", c.expected, b.Bytes()) + }) + } +} + +func TestMarshal_Error(t *testing.T) { + testCases := map[string]struct { + input interface{} + err error + }{ + "InvalidInput": { + struct{}{}, + ErrInvalidType, + }, + "InvalidElementName": { + &struct { + Invalid uint64 `ebml:"Invalid"` + }{}, + ErrUnknownElementName, + }, + "InvalidMapKey": { + &map[int]interface{}{1: "test"}, + ErrNonStringMapKey, + }, + "InvalidType": { + &[]int{}, + ErrIncompatibleType, + }, + } + for n, c := range testCases { + t.Run(n, func(t *testing.T) { + var b bytes.Buffer + if err := Marshal(c.input, &b); !errs.Is(err, c.err) { + t.Fatalf("Expected error: '%v', got: '%v'", c.err, err) + } + }) + } +} + +func TestMarshal_OptionError(t *testing.T) { + errExpected := errors.New("an error") + if err := Marshal(&struct{}{}, &bytes.Buffer{}, + func(*MarshalOptions) error { + return errExpected + }, + ); err != errExpected { + t.Errorf("Expected error against failing MarshalOption: '%v', got: '%v'", errExpected, err) + } +} + +func TestMarshal_WriterError(t *testing.T) { + type EBMLHeader struct { + DocTypeVersion uint64 `ebml:"EBMLDocTypeVersion"` // 2 + 1 + 1 bytes + DocTypeVersion2 uint64 `ebml:"EBMLDocTypeVersion,size=unknown"` // 2 + 8 + 8 bytes + } // 22 bytes + s := struct { + Header EBMLHeader `ebml:"EBML"` // 4 + 1 + 22 bytes + Header2 EBMLHeader `ebml:"EBML,size=unknown"` // 4 + 8 + 22 bytes + }{} // 61 bytes + + for l := 0; l < 61; l++ { + if err := Marshal(&s, &limitedDummyWriter{limit: l}); !errs.Is(err, bytes.ErrTooLarge) { + t.Errorf("Expected error against too large data (Writer size limit: %d): '%v', got '%v'", + l, bytes.ErrTooLarge, err, + ) + } + } +} + +func TestMarshal_EncodeError(t *testing.T) { + s := struct { + SimpleBlock Block + }{ + SimpleBlock: Block{ + Lacing: LacingFixed, + Data: [][]byte{{0x01}, {0x01, 0x02}}, + }, + } + if err := Marshal(&s, &bytes.Buffer{}); !errs.Is(err, ErrUnevenFixedLace) { + t.Errorf("Expected error on encoding uneven fixed lace Block: '%v', got: '%v'", + ErrUnevenFixedLace, err) + } +} + +func TestMarshal_WithWriteHooks(t *testing.T) { + type DummyCluster struct { + Timecode uint64 `ebml:"Timecode"` // 2 + 1 + 1 bytes + } + s := struct { + Header struct { + DocTypeVersion uint64 `ebml:"EBMLDocTypeVersion"` // 2 + 1 + 1 bytes + } `ebml:"EBML"` // 4 + 1 + 4 bytes + Segment struct { + Cluster []DummyCluster `ebml:"Cluster,size=unknown"` // 4 + 8 + 4 bytes + } `ebml:"Segment,size=unknown"` // 4 + 8 + (16 * n) bytes + }{} + s.Segment.Cluster = make([]DummyCluster, 2) + + m := make(map[string][]*Element) + hook := withElementMap(m) + if err := Marshal(&s, &bytes.Buffer{}, WithElementWriteHooks(hook)); err != nil { + t.Errorf("Unexpected error: '%v'", err) + } + + expected := map[string][]uint64{ + "EBML": {0}, + "EBML.EBMLDocTypeVersion": {4}, + "Segment": {9}, + "Segment.Cluster": {21, 36}, + "Segment.Cluster.Timecode": {33, 48}, + } + posMap := elementPositionMap(m) + if !reflect.DeepEqual(expected, posMap) { + t.Errorf("Unexpected write hook positions, \nexpected: %v, \n got: %v", expected, posMap) + } + checkTarget := "Segment.Cluster.Timecode" + switch { + case len(m[checkTarget]) != 2: + t.Fatalf("%s write hook should be called twice, but called %d times", + checkTarget, len(m[checkTarget])) + case m[checkTarget][0].Type != ElementTimecode: + t.Fatalf("ElementType of %s should be %s, got %s", + checkTarget, ElementTimecode, m[checkTarget][0].Type) + } + switch v, ok := m[checkTarget][0].Value.(uint64); { + case !ok: + t.Errorf("Invalid type of data: %T", v) + case v != 0: + t.Errorf("The value should be 0, got %d", v) + } +} + +func ExampleMarshal() { + type EBMLHeader struct { + DocType string `ebml:"EBMLDocType"` + DocTypeVersion uint64 `ebml:"EBMLDocTypeVersion"` + DocTypeReadVersion uint64 `ebml:"EBMLDocTypeReadVersion"` + } + type TestEBML struct { + Header EBMLHeader `ebml:"EBML"` + } + s := TestEBML{ + Header: EBMLHeader{ + DocType: "webm", + DocTypeVersion: 2, + DocTypeReadVersion: 2, + }, + } + + var b bytes.Buffer + if err := Marshal(&s, &b); err != nil { + panic(err) + } + for _, b := range b.Bytes() { + fmt.Printf("0x%02x, ", int(b)) + } + // Output: + // 0x1a, 0x45, 0xdf, 0xa3, 0x8f, 0x42, 0x82, 0x84, 0x77, 0x65, 0x62, 0x6d, 0x42, 0x87, 0x81, 0x02, 0x42, 0x85, 0x81, 0x02, +} + +func ExampleWithDataSizeLen() { + type EBMLHeader struct { + DocType string `ebml:"EBMLDocType"` + DocTypeVersion uint64 `ebml:"EBMLDocTypeVersion"` + DocTypeReadVersion uint64 `ebml:"EBMLDocTypeReadVersion"` + } + type TestEBML struct { + Header EBMLHeader `ebml:"EBML"` + } + s := TestEBML{ + Header: EBMLHeader{ + DocType: "webm", + DocTypeVersion: 2, + DocTypeReadVersion: 2, + }, + } + + var b bytes.Buffer + if err := Marshal(&s, &b, WithDataSizeLen(2)); err != nil { + panic(err) + } + for _, b := range b.Bytes() { + fmt.Printf("0x%02x, ", int(b)) + } + // Output: + // 0x1a, 0x45, 0xdf, 0xa3, 0x40, 0x12, 0x42, 0x82, 0x40, 0x04, 0x77, 0x65, 0x62, 0x6d, 0x42, 0x87, 0x40, 0x01, 0x02, 0x42, 0x85, 0x40, 0x01, 0x02, +} + +func TestMarshal_Tag(t *testing.T) { + tagged := struct { + DocCustomNamedType string `ebml:"EBMLDocType"` + }{ + DocCustomNamedType: "hoge", + } + untagged := struct { + EBMLDocType string + }{ + EBMLDocType: "hoge", + } + + var bTagged, bUntagged bytes.Buffer + if err := Marshal(&tagged, &bTagged); err != nil { + t.Fatalf("Unexpected error: '%v'", err) + } + if err := Marshal(&untagged, &bUntagged); err != nil { + t.Fatalf("Unexpected error: '%v'", err) + } + + if !bytes.Equal(bTagged.Bytes(), bUntagged.Bytes()) { + t.Errorf("Tagged struct and untagged struct must be marshal-ed to same binary, tagged: %v, untagged: %v", bTagged.Bytes(), bUntagged.Bytes()) + } +} + +func TestMarshal_InvalidTag(t *testing.T) { + input := struct { + DocCustomNamedType string `ebml:"EBMLDocType,invalidtag"` + }{ + DocCustomNamedType: "hoge", + } + + var buf bytes.Buffer + if err := Marshal(&input, &buf); !errs.Is(err, ErrInvalidTag) { + t.Errorf("Expected error against invalid tag: '%v', got: '%v'", ErrInvalidTag, err) + } +} + +func TestMarshal_Chan(t *testing.T) { + expected := []byte{ + 0x18, 0x53, 0x80, 0x67, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x1F, 0x43, 0xB6, 0x75, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xE7, 0x81, 0x01, + 0x1F, 0x43, 0xB6, 0x75, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xE7, 0x81, 0x02, + } + type Cluster struct { + Timecode uint64 `ebml:"Timecode"` + } + + t.Run("ChanStruct", func(t *testing.T) { + ch := make(chan Cluster, 100) + input := &struct { + Segment struct { + Cluster chan Cluster `ebml:"Cluster,size=unknown"` + } `ebml:"Segment,size=unknown"` + }{} + input.Segment.Cluster = ch + ch <- Cluster{Timecode: 0x01} + ch <- Cluster{Timecode: 0x02} + close(ch) + + var b bytes.Buffer + if err := Marshal(input, &b); err != nil { + t.Fatalf("Unexpected error: '%v'", err) + } + if !bytes.Equal(expected, b.Bytes()) { + t.Errorf("Marshaled binary doesn't match:\n expected: %v,\n got: %v", expected, b.Bytes()) + } + }) + t.Run("ChanStructPtr", func(t *testing.T) { + input := &struct { + Segment struct { + Cluster chan *Cluster `ebml:"Cluster,size=unknown"` + } `ebml:"Segment,size=unknown"` + }{} + + t.Run("Valid", func(t *testing.T) { + ch := make(chan *Cluster, 100) + input.Segment.Cluster = ch + ch <- &Cluster{Timecode: 0x01} + ch <- &Cluster{Timecode: 0x02} + close(ch) + + var b bytes.Buffer + if err := Marshal(input, &b); err != nil { + t.Fatalf("Unexpected error: '%v'", err) + } + if !bytes.Equal(expected, b.Bytes()) { + t.Errorf("Marshaled binary doesn't match:\n expected: %v,\n got: %v", expected, b.Bytes()) + } + }) + t.Run("Nil", func(t *testing.T) { + ch := make(chan *Cluster, 100) + input.Segment.Cluster = ch + ch <- nil + close(ch) + + if err := Marshal(input, &bytes.Buffer{}); !errs.Is(err, ErrIncompatibleType) { + t.Fatalf("Expected error: '%v', got: '%v'", ErrIncompatibleType, err) + } + }) + }) + t.Run("ChanStructSlice", func(t *testing.T) { + input := &struct { + Segment struct { + Cluster chan []Cluster `ebml:"Cluster,size=unknown"` + } `ebml:"Segment,size=unknown"` + }{} + ch := make(chan []Cluster, 100) + input.Segment.Cluster = ch + ch <- make([]Cluster, 2) + close(ch) + + if err := Marshal(input, &bytes.Buffer{}); !errs.Is(err, ErrIncompatibleType) { + t.Fatalf("Expected error: '%v', got: '%v'", ErrIncompatibleType, err) + } + }) +} + +func TestMarshal_Func(t *testing.T) { + expected := []byte{ + 0x18, 0x53, 0x80, 0x67, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x1F, 0x43, 0xB6, 0x75, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xE7, 0x81, 0x01, + } + type Cluster struct { + Timecode uint64 `ebml:"Timecode"` + } + + t.Run("FuncStruct", func(t *testing.T) { + input := &struct { + Segment struct { + Cluster func() Cluster `ebml:"Cluster,size=unknown"` + } `ebml:"Segment,size=unknown"` + }{} + input.Segment.Cluster = func() Cluster { + return Cluster{Timecode: 0x01} + } + + var b bytes.Buffer + if err := Marshal(input, &b); err != nil { + t.Fatalf("Unexpected error: '%v'", err) + } + if !bytes.Equal(expected, b.Bytes()) { + t.Errorf("Marshaled binary doesn't match:\n expected: %v,\n got: %v", expected, b.Bytes()) + } + }) + t.Run("FuncStructPtr", func(t *testing.T) { + input := &struct { + Segment struct { + Cluster func() *Cluster `ebml:"Cluster,size=unknown"` + } `ebml:"Segment,size=unknown"` + }{} + + t.Run("Valid", func(t *testing.T) { + input.Segment.Cluster = func() *Cluster { + return &Cluster{Timecode: 0x01} + } + + var b bytes.Buffer + if err := Marshal(input, &b); err != nil { + t.Fatalf("Unexpected error: '%v'", err) + } + if !bytes.Equal(expected, b.Bytes()) { + t.Errorf("Marshaled binary doesn't match:\n expected: %v,\n got: %v", expected, b.Bytes()) + } + }) + t.Run("Nil", func(t *testing.T) { + input.Segment.Cluster = func() *Cluster { + return nil + } + + if err := Marshal(input, &bytes.Buffer{}); !errs.Is(err, ErrIncompatibleType) { + t.Fatalf("Expected error: '%v', got: '%v'", ErrIncompatibleType, err) + } + }) + }) + t.Run("FuncStructSlice", func(t *testing.T) { + input := &struct { + Segment struct { + Cluster func() []Cluster `ebml:"Cluster,size=unknown"` + } `ebml:"Segment,size=unknown"` + }{} + input.Segment.Cluster = func() []Cluster { + return []Cluster{ + {Timecode: 0x01}, + {Timecode: 0x02}, + } + } + + var b bytes.Buffer + if err := Marshal(input, &b); err != nil { + t.Fatalf("Unexpected error: '%v'", err) + } + expected := []byte{ + 0x18, 0x53, 0x80, 0x67, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x1F, 0x43, 0xB6, 0x75, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xE7, 0x81, 0x01, + 0x1F, 0x43, 0xB6, 0x75, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xE7, 0x81, 0x02, + } + if !bytes.Equal(expected, b.Bytes()) { + t.Errorf("Marshaled binary doesn't match:\n expected: %v,\n got: %v", expected, b.Bytes()) + } + }) + t.Run("FuncWithError", func(t *testing.T) { + input := &struct { + Segment struct { + Cluster func() (Cluster, error) `ebml:"Cluster,size=unknown"` + } `ebml:"Segment,size=unknown"` + }{} + + t.Run("Valid", func(t *testing.T) { + input.Segment.Cluster = func() (Cluster, error) { + return Cluster{Timecode: 0x01}, nil + } + + var b bytes.Buffer + if err := Marshal(input, &b); err != nil { + t.Fatalf("Unexpected error: '%v'", err) + } + if !bytes.Equal(expected, b.Bytes()) { + t.Errorf("Marshaled binary doesn't match:\n expected: %v,\n got: %v", expected, b.Bytes()) + } + }) + t.Run("Error", func(t *testing.T) { + expectedErr := errors.New("an error") + input.Segment.Cluster = func() (Cluster, error) { + return Cluster{Timecode: 0x01}, expectedErr + } + + if err := Marshal(input, &bytes.Buffer{}); !errs.Is(err, expectedErr) { + t.Fatalf("Expected error: '%v', got: '%v'", expectedErr, err) + } + }) + t.Run("NonErrorType", func(t *testing.T) { + input := &struct { + Segment struct { + Cluster func() (*Cluster, int) `ebml:"Cluster,size=unknown"` + } `ebml:"Segment,size=unknown"` + }{} + input.Segment.Cluster = func() (*Cluster, int) { + return nil, 1 + } + + if err := Marshal(input, &bytes.Buffer{}); !errs.Is(err, ErrIncompatibleType) { + t.Fatalf("Expected error: '%v', got: '%v'", ErrIncompatibleType, err) + } + }) + }) + t.Run("InvalidFunc", func(t *testing.T) { + input := &struct { + Segment struct { + Cluster func() `ebml:"Cluster,size=unknown"` + } `ebml:"Segment,size=unknown"` + }{} + input.Segment.Cluster = func() {} + + if err := Marshal(input, &bytes.Buffer{}); !errs.Is(err, ErrIncompatibleType) { + t.Fatalf("Expected error: '%v', got: '%v'", ErrIncompatibleType, err) + } + }) +} + +func BenchmarkMarshal(b *testing.B) { + type EBMLHeader struct { + DocType string `ebml:"EBMLDocType"` + DocTypeVersion uint64 `ebml:"EBMLDocTypeVersion"` + DocTypeReadVersion uint64 `ebml:"EBMLDocTypeReadVersion"` + } + type TestEBML struct { + Header EBMLHeader `ebml:"EBML"` + } + s := TestEBML{ + Header: EBMLHeader{ + DocType: "webm", + DocTypeVersion: 2, + DocTypeReadVersion: 2, + }, + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + var buf bytes.Buffer + if err := Marshal(&s, &buf); err != nil { + b.Fatalf("Unexpected error: '%v'", err) + } + } +} diff --git a/pkg/ebml-go/matroska_official_test.go b/pkg/ebml-go/matroska_official_test.go new file mode 100644 index 0000000..4d5a93d --- /dev/null +++ b/pkg/ebml-go/matroska_official_test.go @@ -0,0 +1,120 @@ +//go:build matroska_official +// +build matroska_official + +package ebml + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "testing" +) + +const ( + testDataBaseURL = "https://raw.githubusercontent.com/Matroska-Org/matroska-test-files/master/test_files/" + cacheDir = "ebml-go-matroska-official-test-data" +) + +func loadTestData(t *testing.T, file string) ([]byte, error) { + var r io.ReadCloser + var hasCache bool + + cacheFile := filepath.Join(os.TempDir(), cacheDir, file) + if f, err := os.Open(cacheFile); err == nil { + t.Logf("Using cache: %s", cacheFile) + r = f + hasCache = true + } else { + mkvResp, err := http.Get(testDataBaseURL + file) + if err != nil { + return nil, err + } + r = mkvResp.Body + } + + b, err := ioutil.ReadAll(r) + if err != nil { + r.Close() + return nil, err + } + r.Close() + + if !hasCache { + os.MkdirAll(filepath.Join(os.TempDir(), cacheDir), 0755) + if f, err := os.Create(cacheFile); err == nil { + n, err := f.Write(b) + f.Close() + if err != nil || n != len(b) { + os.Remove(cacheFile) + } else { + t.Logf("Saved cache: %s", cacheFile) + } + } + } + + return b, nil +} + +func TestMatroskaOfficial(t *testing.T) { + testData := map[string]struct { + filename string + opts []UnmarshalOption + }{ + "Basic": { + filename: "test1.mkv", + }, + "NonDefaultTimecodeScaleAndAspectRatio": { + filename: "test2.mkv", + }, + "HeaderStrippingAndStandardBlock": { + filename: "test3.mkv", + }, + "LiveStreamRecording": { + filename: "test4.mkv", + opts: []UnmarshalOption{WithIgnoreUnknown(true)}, + }, + "MultipleAudioSubtitles": { + filename: "test5.mkv", + }, + "DifferentEBMLHeadSizesAndCueLessSeeking": { + filename: "test6.mkv", + }, + "ExtraUnknownJunkElementsDamaged": { + filename: "test7.mkv", + opts: []UnmarshalOption{WithIgnoreUnknown(true)}, + }, + "AudioGap": { + filename: "test8.mkv", + }, + } + for name, tt := range testData { + tt := tt + t.Run(name, func(t *testing.T) { + mkvRaw, err := loadTestData(t, tt.filename) + if err != nil { + t.Fatal(err) + } + + var dump string + for i := 0; i < 16 && i < len(mkvRaw); i++ { + dump += fmt.Sprintf("%02x ", mkvRaw[i]) + } + t.Logf("dump: %s", dump) + + var mkv map[string]interface{} + if err := Unmarshal(bytes.NewReader(mkvRaw), &mkv, tt.opts...); err != nil { + t.Fatalf("Failed to unmarshal: '%v'", err) + } + txt := fmt.Sprintf("%+v", mkv) + if len(txt) > 512 { + t.Logf("result: %s...", txt[:512]) + } else { + t.Logf("result: %s", txt) + } + }) + } +} diff --git a/pkg/ebml-go/mkv/const.go b/pkg/ebml-go/mkv/const.go new file mode 100644 index 0000000..243ced6 --- /dev/null +++ b/pkg/ebml-go/mkv/const.go @@ -0,0 +1,34 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mkv + +var ( + // DefaultEBMLHeader is the default EBML header used by BlockWriter. + DefaultEBMLHeader = &EBMLHeader{ + EBMLVersion: 1, + EBMLReadVersion: 1, + EBMLMaxIDLength: 4, + EBMLMaxSizeLength: 8, + DocType: "matroska", + DocTypeVersion: 4, // May contain v4 elements, + DocTypeReadVersion: 2, // and playable by parsing v2 elements. + } + // DefaultSegmentInfo is the default Segment.Info used by BlockWriter. + DefaultSegmentInfo = &Info{ + TimecodeScale: 1000000, // 1ms + MuxingApp: "ebml-go.mkv.BlockWriter", + WritingApp: "ebml-go.mkv.BlockWriter", + } +) diff --git a/pkg/ebml-go/mkv/mkv.go b/pkg/ebml-go/mkv/mkv.go new file mode 100644 index 0000000..cff2a0f --- /dev/null +++ b/pkg/ebml-go/mkv/mkv.go @@ -0,0 +1,42 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package mkv provides the Matroska multimedia writer. +// +// The package implements block data writer for multi-track Matroska container. +package mkv + +import ( + "time" +) + +// EBMLHeader represents EBML header struct. +type EBMLHeader struct { + EBMLVersion uint64 `ebml:"EBMLVersion"` + EBMLReadVersion uint64 `ebml:"EBMLReadVersion"` + EBMLMaxIDLength uint64 `ebml:"EBMLMaxIDLength"` + EBMLMaxSizeLength uint64 `ebml:"EBMLMaxSizeLength"` + DocType string `ebml:"EBMLDocType"` + DocTypeVersion uint64 `ebml:"EBMLDocTypeVersion"` + DocTypeReadVersion uint64 `ebml:"EBMLDocTypeReadVersion"` +} + +// Info represents Info element struct. +type Info struct { + TimecodeScale uint64 `ebml:"TimecodeScale"` + MuxingApp string `ebml:"MuxingApp,omitempty"` + WritingApp string `ebml:"WritingApp,omitempty"` + Duration float64 `ebml:"Duration,omitempty"` + DateUTC time.Time `ebml:"DateUTC,omitempty"` +} diff --git a/pkg/ebml-go/mkvcore/blockreader.go b/pkg/ebml-go/mkvcore/blockreader.go new file mode 100644 index 0000000..a63fee4 --- /dev/null +++ b/pkg/ebml-go/mkvcore/blockreader.go @@ -0,0 +1,171 @@ +// Copyright 2020-2021 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mkvcore + +import ( + "io" + + "github.com/at-wat/ebml-go" +) + +type blockReader struct { + f chan *frame + closed chan struct{} + trackEntry TrackEntry +} + +func (r *blockReader) Read() (b []byte, keyframe bool, timestamp int64, err error) { + frame, ok := <-r.f + if !ok { + return nil, false, 0, io.EOF + } + return frame.b, frame.keyframe, frame.timestamp, nil +} + +func (r *blockReader) Close() error { + close(r.closed) + return nil +} + +func (r *blockReader) TrackEntry() TrackEntry { + return r.trackEntry +} + +// NewSimpleBlockReader creates BlockReadCloserWithTrackEntry for each track specified as tracks argument. +// It reads SimpleBlock-s and BlockGroup.Block-s. Any optional data in BlockGroup are dropped. +// If you need full data, consider implementing a custom reader using ebml.Unmarshal. +// +// Note that, keyframe flag from BlockGroup.Block may be incorrect. +// If you have knowledge about this, please consider fixing it. +func NewSimpleBlockReader(r io.Reader, opts ...BlockReaderOption) ([]BlockReadCloserWithTrackEntry, error) { + options := &BlockReaderOptions{ + BlockReadWriterOptions: BlockReadWriterOptions{ + onFatal: func(err error) { + panic(err) + }, + }, + } + for _, o := range opts { + if err := o.ApplyToBlockReaderOptions(options); err != nil { + return nil, err + } + } + + var header struct { + Segment struct { + Tracks struct { + TrackEntry []TrackEntry + } `ebml:"Tracks,stop"` + } + } + switch err := ebml.Unmarshal(r, &header, options.unmarshalOpts...); err { + case ebml.ErrReadStopped: + default: + return nil, err + } + + var ws []BlockReadCloserWithTrackEntry + br := make(map[uint64]*blockReader) + + for _, t := range header.Segment.Tracks.TrackEntry { + r := &blockReader{ + f: make(chan *frame), + closed: make(chan struct{}), + trackEntry: t, + } + ws = append(ws, r) + br[t.TrackNumber] = r + } + + type blockGroup struct { + Block ebml.Block + ReferencePriority uint64 + } + type clusterReader struct { + Timecode uint64 + SimpleBlock chan ebml.Block + BlockGroup chan blockGroup + } + blockCh := make(chan ebml.Block) + blockGroupCh := make(chan blockGroup) + c := struct { + Cluster clusterReader + }{ + Cluster: clusterReader{ + SimpleBlock: blockCh, + BlockGroup: blockGroupCh, + }, + } + go func() { + blockCh := blockCh + blockGroupCh := blockGroupCh + L_READ: + for { + var b *ebml.Block + select { + case block, ok := <-blockCh: + if !ok { + blockCh = nil + if blockGroupCh == nil { + break L_READ + } + continue + } + b = &block + case bg, ok := <-blockGroupCh: + if !ok { + blockGroupCh = nil + if blockCh == nil { + break L_READ + } + continue + } + b = &bg.Block + // FIXME: This may be wrong. + // ReferencePriority == 0 means that the frame is not referenced. + b.Keyframe = bg.ReferencePriority != 0 + } + r := br[b.TrackNumber] + for l := range b.Data { + frame := &frame{ + trackNumber: b.TrackNumber, + keyframe: b.Keyframe, + timestamp: int64(c.Cluster.Timecode) + int64(b.Timecode), + b: b.Data[l], + } + select { + case r.f <- frame: + case <-r.closed: + } + } + } + for k := range br { + close(br[k].f) + } + }() + go func() { + defer func() { + close(blockCh) + close(blockGroupCh) + }() + if err := ebml.Unmarshal(r, &c, options.unmarshalOpts...); err != nil { + if options.onFatal != nil { + options.onFatal(err) + } + } + }() + + return ws, nil +} diff --git a/pkg/ebml-go/mkvcore/blockreader_test.go b/pkg/ebml-go/mkvcore/blockreader_test.go new file mode 100644 index 0000000..8edf539 --- /dev/null +++ b/pkg/ebml-go/mkvcore/blockreader_test.go @@ -0,0 +1,440 @@ +// Copyright 2020-2021 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mkvcore + +import ( + "bytes" + "errors" + "io" + "reflect" + "sync" + "testing" + "time" + + "github.com/at-wat/ebml-go" + "github.com/at-wat/ebml-go/internal/buffercloser" + "github.com/at-wat/ebml-go/internal/errs" +) + +func TestBlockReader(t *testing.T) { + type testMkvHeader struct { + Segment flexSegment `ebml:"Segment"` + } + testCases := map[string]struct { + input testMkvHeader + expectedTrackEntries []TrackEntry + expected [][]frame + }{ + "TwoTracks": { + input: testMkvHeader{ + Segment: flexSegment{ + Tracks: flexTracks{TrackEntry: []interface{}{ + map[string]interface{}{"TrackNumber": uint(1)}, + map[string]interface{}{"TrackNumber": uint(2)}, + }}, + Cluster: []simpleBlockCluster{ + { + Timecode: uint64(100), + SimpleBlock: []ebml.Block{ + { + TrackNumber: 1, + Timecode: int16(-10), + Keyframe: false, + Data: [][]byte{{0x01, 0x02}}, + }, + { + TrackNumber: 2, + Timecode: int16(10), + Keyframe: true, + Data: [][]byte{{0x03, 0x04, 0x05}}, + }, + { + TrackNumber: 1, + Timecode: int16(30), + Keyframe: true, + Data: [][]byte{{0x06}}, + }, + }, + }, + { + Timecode: uint64(30), + PrevSize: uint64(39), + }, + }, + }, + }, + expectedTrackEntries: []TrackEntry{ + {TrackNumber: 1}, + {TrackNumber: 2}, + }, + expected: [][]frame{ + { + {keyframe: false, timestamp: 90, b: []byte{0x01, 0x02}}, + {keyframe: true, timestamp: 130, b: []byte{0x06}}, + }, + { + {keyframe: true, timestamp: 110, b: []byte{0x03, 0x04, 0x05}}, + }, + }, + }, + "SimpleBlockAndBlock": { + input: testMkvHeader{ + Segment: flexSegment{ + Tracks: flexTracks{TrackEntry: []interface{}{ + map[string]interface{}{"TrackNumber": uint(1)}, + map[string]interface{}{"TrackNumber": uint(2)}, + }}, + Cluster: []simpleBlockCluster{ + { + Timecode: uint64(100), + SimpleBlock: []ebml.Block{ + { + TrackNumber: 1, + Timecode: int16(-10), + Keyframe: false, + Data: [][]byte{{0x01, 0x02}}, + }, + { + TrackNumber: 2, + Timecode: int16(10), + Keyframe: true, + Data: [][]byte{{0x03, 0x04, 0x05}}, + }, + }, + BlockGroup: []simpleBlockGroup{ + { + Block: []ebml.Block{ + { + TrackNumber: 1, + Timecode: int16(30), + Data: [][]byte{{0x06}}, + }, + }, + ReferencePriority: 1, + }, + { + Block: []ebml.Block{ + { + TrackNumber: 2, + Timecode: int16(40), + Data: [][]byte{{0x07}}, + }, + }, + }, + }, + }, + }, + }, + }, + expectedTrackEntries: []TrackEntry{ + {TrackNumber: 1}, + {TrackNumber: 2}, + }, + expected: [][]frame{ + { + {keyframe: false, timestamp: 90, b: []byte{0x01, 0x02}}, + {keyframe: true, timestamp: 130, b: []byte{0x06}}, + }, + { + {keyframe: true, timestamp: 110, b: []byte{0x03, 0x04, 0x05}}, + {keyframe: false, timestamp: 140, b: []byte{0x07}}, + }, + }, + }, + "NoBlock": { + input: testMkvHeader{ + Segment: flexSegment{ + Tracks: flexTracks{TrackEntry: []interface{}{ + map[string]interface{}{"TrackNumber": uint(1)}, + map[string]interface{}{"TrackNumber": uint(2)}, + }}, + Cluster: []simpleBlockCluster{}, + }, + }, + expectedTrackEntries: []TrackEntry{ + {TrackNumber: 1}, + {TrackNumber: 2}, + }, + expected: [][]frame{{}, {}}, + }, + "NoCluster": { + input: testMkvHeader{ + Segment: flexSegment{ + Tracks: flexTracks{TrackEntry: []interface{}{ + map[string]interface{}{"TrackNumber": uint(1)}, + map[string]interface{}{"TrackNumber": uint(2)}, + }}, + }, + }, + expectedTrackEntries: []TrackEntry{ + {TrackNumber: 1}, + {TrackNumber: 2}, + }, + expected: [][]frame{{}, {}}, + }, + } + + for name, testCase := range testCases { + testCase := testCase + t.Run(name, func(t *testing.T) { + buf := buffercloser.New() + if err := ebml.Marshal(&testCase.input, buf); err != nil { + t.Fatalf("Failed to marshal test data: '%v'", err) + } + buf.Close() + + rs, err := NewSimpleBlockReader(bytes.NewReader(buf.Bytes())) + if err != nil { + t.Fatalf("Failed to create BlockReader: '%v'", err) + } + + if len(rs) != len(testCase.expected) { + t.Fatalf("Number of the returned writer (%d) must be same as the number of TrackEntry (%d)", len(rs), len(testCase.expected)) + } + + for i, r := range rs { + if !reflect.DeepEqual(testCase.expectedTrackEntries[i], r.TrackEntry()) { + t.Errorf("Expected TrackEntry[%d]: %v, got: %v", i, testCase.expectedTrackEntries[i], r.TrackEntry()) + } + } + + var wg sync.WaitGroup + wg.Add(len(testCase.expected)) + + for i, dd := range testCase.expected { + i, dd := i, dd + go func() { + defer wg.Done() + + for _, d := range dd { + buf, keyframe, timestamp, err := rs[i].Read() + if err != nil { + t.Errorf("Failed to Read: '%v'", err) + } + if keyframe != d.keyframe { + t.Errorf("Expected keyframe: %v, got: %v", d.keyframe, keyframe) + } + if timestamp != d.timestamp { + t.Errorf("Expected timestamp: %v, got: %v", d.timestamp, timestamp) + } + if !bytes.Equal(buf, d.b) { + t.Errorf("Expected bytes: %v, got: %v", d.b, buf) + } + } + if _, _, _, err := rs[i].Read(); err != io.EOF { + t.Errorf("Expected: EOF, got: %v", err) + } + if err := rs[i].Close(); err != nil { + t.Errorf("Unexpected error: %v", err) + } + }() + } + + wg.Wait() + }) + } +} + +var errTimeout = errors.New("timeout") + +func readWithTimeout(r BlockReader) error { + errCh := make(chan error) + go func() { + _, _, _, err := r.Read() + errCh <- err + }() + + select { + case err := <-errCh: + return err + case <-time.After(time.Second): + return errTimeout + } +} + +func TestBlockReader_Close(t *testing.T) { + type testMkvHeader struct { + Segment flexSegment `ebml:"Segment"` + } + input := testMkvHeader{ + Segment: flexSegment{ + Tracks: flexTracks{TrackEntry: []interface{}{ + map[string]interface{}{"TrackNumber": uint(1)}, + map[string]interface{}{"TrackNumber": uint(2)}, + }}, + Cluster: []simpleBlockCluster{ + { + SimpleBlock: []ebml.Block{ + {TrackNumber: 1, Data: [][]byte{{0x01}}}, + {TrackNumber: 2, Data: [][]byte{{0x02}}}, + {TrackNumber: 1, Data: [][]byte{{0x03}}}, + }, + }, + }, + }, + } + + buf := buffercloser.New() + if err := ebml.Marshal(&input, buf); err != nil { + t.Fatalf("Failed to marshal test data: '%v'", err) + } + buf.Close() + + rs, err := NewSimpleBlockReader(bytes.NewReader(buf.Bytes())) + if err != nil { + t.Fatalf("Failed to create BlockReader: '%v'", err) + } + + if len(rs) != 2 { + t.Fatalf("Number of the returned writer (%d) must be same as the number of TrackEntry (%d)", len(rs), 2) + } + + if err := rs[0].Close(); err != nil { + t.Fatalf("Unexpected Close error: '%v'", err) + } + + if err := readWithTimeout(rs[1]); err != nil { + t.Fatalf("Unexpected Read error: '%v'", err) + } +} + +func TestBlockReader_FailingOptions(t *testing.T) { + errDummy0 := errors.New("an error 0") + errDummy1 := errors.New("an error 1") + + cases := map[string]struct { + opts []BlockReaderOption + err error + }{ + "ReaderOptionError": { + opts: []BlockReaderOption{ + BlockReaderOptionFn(func(*BlockReaderOptions) error { return errDummy0 }), + }, + err: errDummy0, + }, + "UnmarshalOptionError": { + opts: []BlockReaderOption{ + WithUnmarshalOptions( + func(*ebml.UnmarshalOptions) error { return errDummy1 }, + ), + }, + err: errDummy1, + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + buf := bytes.NewReader([]byte{}) + _, err := NewSimpleBlockReader(buf, c.opts...) + if !errs.Is(err, c.err) { + t.Errorf("Expected error: '%v', got: '%v'", c.err, err) + } + }) + } +} + +func TestBlockReader_WithUnmarshalOptions(t *testing.T) { + testCases := map[string]struct { + opts []BlockReaderOption + err error + nReaders int + }{ + "Default": { + err: ebml.ErrUnknownElement, + }, + "IgnoreUnknown": { + opts: []BlockReaderOption{ + WithUnmarshalOptions(ebml.WithIgnoreUnknown(true)), + }, + nReaders: 1, + }, + } + + for name, testCase := range testCases { + testCase := testCase + t.Run(name, func(t *testing.T) { + testBinary := []byte{ + 0x18, 0x53, 0x80, 0x67, 0xFF, // Segment + 0x16, 0x54, 0xae, 0x6b, 0x87, // Tracks + 0x81, 0x81, // 0x81 is not defined in Matroska v4 + 0xae, 0x83, // TrackEntry[0] + 0xd7, 0x81, 0x01, // TrackNumber=1 + 0x1F, 0x43, 0xB6, 0x75, 0xFF, // Cluster + 0xE7, 0x81, 0x00, // Timecode + 0xA3, 0x86, 0x81, 0x00, 0x00, 0x88, 0xAA, 0xCC, // SimpleBlock + } + + rs, err := NewSimpleBlockReader( + bytes.NewReader(testBinary), + testCase.opts..., + ) + if !errs.Is(err, testCase.err) { + if testCase.err != nil { + t.Fatalf("Expected error: '%v', got: '%v'", testCase.err, err) + } else { + t.Fatalf("Unexpected error: '%v'", err) + } + } + + if len(rs) != testCase.nReaders { + t.Fatalf("Number of the returned writer (%d) must be same as the number of TrackEntry (%d)", len(rs), testCase.nReaders) + } + }) + } +} + +func TestBlockReader_WithOnFatalHandler(t *testing.T) { + testBinary := []byte{ + 0x18, 0x53, 0x80, 0x67, 0xFF, // Segment + 0x16, 0x54, 0xae, 0x6b, 0x85, // Tracks + 0xae, 0x83, // TrackEntry[0] + 0xd7, 0x81, 0x01, // TrackNumber=1 + 0x1F, 0x43, 0xB6, 0x75, 0xFF, // Cluster + 0x81, // 0x81 is not defined in Matroska v4 + 0xE7, 0x81, 0x00, // Timecode + } + + chFatal := make(chan error) + rs, err := NewSimpleBlockReader( + bytes.NewReader(testBinary), + WithOnFatalHandler(func(err error) { + chFatal <- err + }), + ) + if err != nil { + t.Fatalf("Unexpected error: '%v'", err) + } + + if len(rs) != 1 { + t.Fatalf("Number of the returned writer (%d) must be same as the number of TrackEntry (%d)", len(rs), 1) + } + + go func() { + if err := readWithTimeout(rs[0]); err != io.EOF { + t.Errorf("Unexpected Read error: '%v'", err) + } + close(chFatal) + }() + + select { + case err := <-chFatal: + // Expected error + if !errs.Is(err, ebml.ErrUnknownElement) { + t.Errorf("Expected error: '%v', got: '%v'", ebml.ErrUnknownElement, err) + } + case <-time.After(time.Second): + t.Error("Timeout") + } +} diff --git a/pkg/ebml-go/mkvcore/blockwriter.go b/pkg/ebml-go/mkvcore/blockwriter.go new file mode 100644 index 0000000..fcd32b9 --- /dev/null +++ b/pkg/ebml-go/mkvcore/blockwriter.go @@ -0,0 +1,249 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mkvcore + +import ( + "errors" + "io" + "sync" + + "github.com/at-wat/ebml-go" +) + +// ErrIgnoreOldFrame means that a frame has too old timestamp and ignored. +var ErrIgnoreOldFrame = errors.New("too old frame") + +type blockWriter struct { + trackNumber uint64 + f chan *frame + wg *sync.WaitGroup + fin chan struct{} +} + +type frame struct { + trackNumber uint64 + keyframe bool + timestamp int64 + b []byte +} + +func (w *blockWriter) Write(keyframe bool, timestamp int64, b []byte) (int, error) { + w.f <- &frame{ + trackNumber: w.trackNumber, + keyframe: keyframe, + timestamp: timestamp, + b: b, + } + return len(b), nil +} + +func (w *blockWriter) Close() error { + w.wg.Done() + + // If it is the last writer, block until closing output writer. + w.fin <- struct{}{} + + return nil +} + +// TrackDescription stores track number and its TrackEntry struct. +type TrackDescription struct { + TrackNumber uint64 + TrackEntry interface{} +} + +// NewSimpleBlockWriter creates BlockWriteCloser for each track specified as tracks argument. +// Blocks will be written to the writer as EBML SimpleBlocks. +// Given io.WriteCloser will be closed automatically; don't close it by yourself. +// Frames written to each track must be sorted by their timestamp. +func NewSimpleBlockWriter(w0 io.WriteCloser, tracks []TrackDescription, opts ...BlockWriterOption) ([]BlockWriteCloser, error) { + options := &BlockWriterOptions{ + BlockReadWriterOptions: BlockReadWriterOptions{ + onFatal: func(err error) { + panic(err) + }, + }, + ebmlHeader: nil, + segmentInfo: nil, + interceptor: nil, + seekHead: false, + } + for _, o := range opts { + if err := o.ApplyToBlockWriterOptions(options); err != nil { + return nil, err + } + } + + w := &writerWithSizeCount{w: w0} + + header := flexHeader{ + Header: options.ebmlHeader, + Segment: flexSegment{ + Info: options.segmentInfo, + }, + } + for _, t := range tracks { + header.Segment.Tracks.TrackEntry = append(header.Segment.Tracks.TrackEntry, t.TrackEntry) + } + if options.seekHead { + if err := setSeekHead(&header, options.marshalOpts...); err != nil { + return nil, err + } + } + if err := ebml.Marshal(&header, w, options.marshalOpts...); err != nil { + return nil, err + } + + w.Clear() + + ch := make(chan *frame) + fin := make(chan struct{}, len(tracks)-1) + wg := sync.WaitGroup{} + var ws []BlockWriteCloser + var fw []BlockWriter + var fr []BlockReader + + for _, t := range tracks { + wg.Add(1) + var chSrc chan *frame + if options.interceptor == nil { + chSrc = ch + } else { + chSrc = make(chan *frame) + fr = append(fr, &filterReader{chSrc}) + fw = append(fw, &filterWriter{t.TrackNumber, ch}) + } + ws = append(ws, &blockWriter{ + trackNumber: t.TrackNumber, + f: chSrc, + wg: &wg, + fin: fin, + }) + } + + filterFlushed := make(chan struct{}) + if options.interceptor != nil { + go func() { + options.interceptor.Intercept(fr, fw) + close(filterFlushed) + }() + } else { + close(filterFlushed) + } + + closed := make(chan struct{}) + go func() { + wg.Wait() + for _, c := range fr { + c.(*filterReader).close() + } + <-filterFlushed + close(closed) + }() + + tNextCluster := 0x7FFF - options.maxKeyframeInterval + + go func() { + const invalidTimestamp = int64(0x7FFFFFFFFFFFFFFF) + tc0 := invalidTimestamp + tc1 := invalidTimestamp + lastTc := int64(0) + + defer func() { + // Finalize WebM + if tc0 == invalidTimestamp { + // No data written + tc0 = 0 + } + cluster := struct { + Cluster simpleBlockCluster `ebml:"Cluster,size=unknown"` + }{ + Cluster: simpleBlockCluster{ + Timecode: uint64(lastTc - tc0), + PrevSize: uint64(w.Size()), + }, + } + if err := ebml.Marshal(&cluster, w, options.marshalOpts...); err != nil { + if options.onFatal != nil { + options.onFatal(err) + } + } + w.Close() + <-fin // read one data to release blocked Close() + }() + + L_WRITE: + for { + select { + case <-closed: + break L_WRITE + case f := <-ch: + if tc0 == invalidTimestamp { + tc0 = f.timestamp + } + lastTc = f.timestamp + tc := f.timestamp - tc1 + if tc1 == invalidTimestamp || tc >= 0x7FFF || (f.trackNumber == options.mainTrackNumber && tc >= tNextCluster && f.keyframe) { + // Create new Cluster + tc1 = f.timestamp + tc = 0 + + cluster := struct { + Cluster simpleBlockCluster `ebml:"Cluster,size=unknown"` + }{ + Cluster: simpleBlockCluster{ + Timecode: uint64(tc1 - tc0), + PrevSize: uint64(w.Size()), + }, + } + w.Clear() + if err := ebml.Marshal(&cluster, w, options.marshalOpts...); err != nil { + if options.onFatal != nil { + options.onFatal(err) + } + return + } + } + if tc <= -0x7FFF { + // Ignore too old frame + if options.onError != nil { + options.onError(ErrIgnoreOldFrame) + } + continue + } + + b := struct { + Block ebml.Block `ebml:"SimpleBlock"` + }{ + ebml.Block{ + TrackNumber: f.trackNumber, + Timecode: int16(tc), + Keyframe: f.keyframe, + Data: [][]byte{f.b}, + }, + } + // Write SimpleBlock to the file + if err := ebml.Marshal(&b, w, options.marshalOpts...); err != nil { + if options.onFatal != nil { + options.onFatal(err) + } + return + } + } + } + }() + + return ws, nil +} diff --git a/pkg/ebml-go/mkvcore/blockwriter_test.go b/pkg/ebml-go/mkvcore/blockwriter_test.go new file mode 100644 index 0000000..674e9f8 --- /dev/null +++ b/pkg/ebml-go/mkvcore/blockwriter_test.go @@ -0,0 +1,582 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mkvcore + +import ( + "bytes" + "errors" + "reflect" + "sync" + "testing" + "time" + + "github.com/at-wat/ebml-go" + "github.com/at-wat/ebml-go/internal/buffercloser" + "github.com/at-wat/ebml-go/internal/errs" +) + +func TestBlockWriter(t *testing.T) { + buf := buffercloser.New() + + tracks := []TrackDescription{ + {TrackNumber: 1}, + {TrackNumber: 2}, + } + + blockSorter, err := NewMultiTrackBlockSorter(WithMaxDelayedPackets(10), WithSortRule(BlockSorterDropOutdated)) + if err != nil { + t.Fatalf("Failed to create MultiTrackBlockSorter: %v", err) + } + + ws, err := NewSimpleBlockWriter(buf, tracks, + WithBlockInterceptor(blockSorter)) + if err != nil { + t.Fatalf("Failed to create BlockWriter: '%v'", err) + } + + if len(ws) != len(tracks) { + t.Fatalf("Number of the returned writer (%d) must be same as the number of TrackEntry (%d)", len(ws), len(tracks)) + } + + if n, err := ws[1].Write(true, 110, []byte{0x03, 0x04, 0x05}); err != nil { + t.Fatalf("Failed to Write: '%v'", err) + } else if n != 3 { + t.Errorf("Expected return value of BlockWriter.Write: 3, got: %d", n) + } + + if n, err := ws[0].Write(false, 100, []byte{0x01, 0x02}); err != nil { + t.Fatalf("Failed to Write: '%v'", err) + } else if n != 2 { + t.Errorf("Expected return value of BlockWriter.Write: 2, got: %d", n) + } + + // Ignored due to old timestamp + if n, err := ws[0].Write(true, -32769, []byte{0x0A}); err != nil { + t.Fatalf("Failed to Write: '%v'", err) + } else if n != 1 { + t.Errorf("Expected return value of BlockWriter.Write: 1, got: %d", n) + } + + if n, err := ws[0].Write(true, 130, []byte{0x06}); err != nil { + t.Fatalf("Failed to Write: '%v'", err) + } else if n != 1 { + t.Errorf("Expected return value of BlockWriter.Write: 1, got: %d", n) + } + + ws[0].Close() + ws[1].Close() + select { + case <-buf.Closed(): + default: + t.Errorf("Base io.WriteCloser is not closed by BlockWriter") + } + + expected := struct { + Segment flexSegment `ebml:"Segment,size=unknown"` + }{ + Segment: flexSegment{ + Cluster: []simpleBlockCluster{ + { + Timecode: uint64(0), + SimpleBlock: []ebml.Block{ + { + TrackNumber: 1, + Timecode: int16(0), + Keyframe: false, + Data: [][]byte{{0x01, 0x02}}, + }, + { + TrackNumber: 2, + Timecode: int16(10), + Keyframe: true, + Data: [][]byte{{0x03, 0x04, 0x05}}, + }, + { + TrackNumber: 1, + Timecode: int16(30), + Keyframe: true, + Data: [][]byte{{0x06}}, + }, + }, + }, + { + Timecode: uint64(30), + PrevSize: uint64(39), + }, + }, + }, + } + var result struct { + Segment flexSegment `ebml:"Segment,size=unknown"` + } + if err := ebml.Unmarshal(bytes.NewReader(buf.Bytes()), &result); err != nil { + t.Fatalf("Failed to Unmarshal resultant binary: '%v'", err) + } + if !reflect.DeepEqual(expected, result) { + t.Errorf("Unexpected data,\nexpected: %+v\n got: %+v", expected, result) + } +} + +func TestBlockWriter_Options(t *testing.T) { + buf := buffercloser.New() + + ws, err := NewSimpleBlockWriter( + buf, + []TrackDescription{{TrackNumber: 1}}, + WithEBMLHeader(&struct { + DocTypeVersion uint64 `ebml:"EBMLDocTypeVersion"` + }{}), + WithSegmentInfo(nil), + WithMarshalOptions(ebml.WithDataSizeLen(2)), + WithSeekHead(false), + ) + if err != nil { + t.Fatalf("Failed to create BlockWriter: '%v'", err) + } + + if len(ws) != 1 { + t.Fatalf("Number of the returned writer must be 1, got %d", len(ws)) + } + ws[0].Close() + + expectedBytes := []byte{ + 0x1A, 0x45, 0xDF, 0xA3, 0x40, 0x05, + 0x42, 0x87, 0x40, 0x01, 0x00, + 0x18, 0x53, 0x80, 0x67, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x16, 0x54, 0xAE, 0x6B, 0x40, 0x00, + 0x1F, 0x43, 0xB6, 0x75, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xE7, 0x40, 0x01, 0x00, + } + if !bytes.Equal(buf.Bytes(), expectedBytes) { + t.Errorf("Unexpected binary,\nexpected: %+v\n got: %+v", expectedBytes, buf.Bytes()) + } +} + +func TestBlockWriter_FailingOptions(t *testing.T) { + errDummy0 := errors.New("an error 0") + errDummy1 := errors.New("an error 1") + + cases := map[string]struct { + opts []BlockWriterOption + err error + }{ + "WriterOptionError": { + opts: []BlockWriterOption{ + BlockWriterOptionFn(func(*BlockWriterOptions) error { return errDummy0 }), + }, + err: errDummy0, + }, + "MarshalOptionError": { + opts: []BlockWriterOption{ + WithMarshalOptions( + func(*ebml.MarshalOptions) error { return errDummy1 }, + ), + WithSeekHead(false), + }, + err: errDummy1, + }, + "MarshalOptionErrorWithSeekHead": { + opts: []BlockWriterOption{ + WithMarshalOptions( + func(*ebml.MarshalOptions) error { + return errDummy1 + }, + ), + }, + err: errDummy1, + }, + "MaxKeyframeIntervalOptionError": { + opts: []BlockWriterOption{ + WithMaxKeyframeInterval(0, 0), + }, + err: ErrInvalidTrackNumber, + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + buf := buffercloser.New() + _, err := NewSimpleBlockWriter(buf, []TrackDescription{}, c.opts...) + if !errs.Is(err, c.err) { + t.Errorf("Expected error: '%v', got: '%v'", c.err, err) + } + }) + } +} + +type errorWriter struct { + wrote chan struct{} + err error + mu sync.Mutex +} + +func (w *errorWriter) setError(err error) { + w.mu.Lock() + w.err = err + w.mu.Unlock() +} + +func (w *errorWriter) Write(b []byte) (int, error) { + select { + case w.wrote <- struct{}{}: + default: + } + w.mu.Lock() + defer w.mu.Unlock() + if w.err != nil { + return 0, w.err + } + return len(b), nil +} + +func (w *errorWriter) WaitWrite() bool { + select { + case <-w.wrote: + case <-time.After(time.Second): + return false + } + return true +} + +func (w *errorWriter) Close() error { + return nil +} + +func TestBlockWriter_ErrorHandling(t *testing.T) { + + const ( + atBeginning int = iota + atClusterWriting + atFrameWriting + atClosing + ) + + for name, errAt := range map[string]int{ + "ErrorAtBeginning": atBeginning, + "ErrorAtClusterWriting": atClusterWriting, + "ErrorAtFrameWriting": atFrameWriting, + "ErrorAtClosing": atClosing, + } { + t.Run(name, func(t *testing.T) { + chFatal := make(chan error, 1) + chError := make(chan error, 1) + clearErr := func() { + for { + select { + case <-chFatal: + case <-chError: + default: + return + } + } + } + + w := &errorWriter{wrote: make(chan struct{}, 1)} + + if errAt == atBeginning { + w.setError(bytes.ErrTooLarge) + } + clearErr() + ws, err := NewSimpleBlockWriter( + w, + []TrackDescription{{TrackNumber: 1}}, + WithOnErrorHandler(func(err error) { chError <- err }), + WithOnFatalHandler(func(err error) { chFatal <- err }), + ) + if err != nil { + if errAt == atBeginning { + if !errs.Is(err, bytes.ErrTooLarge) { + t.Fatalf("Expected error: '%v', got: '%v'", bytes.ErrTooLarge, err) + } + return + } + t.Fatalf("Failed to create SimpleWriter: '%v'", err) + } + + if len(ws) != 1 { + t.Fatalf("Number of the returned writer must be 1, got %d", len(ws)) + } + + if errAt == atClusterWriting { + w.setError(bytes.ErrTooLarge) + } + clearErr() + if _, err := ws[0].Write(false, 100, []byte{0x01, 0x02}); err != nil { + t.Fatalf("Failed to Write: '%v'", err) + } + if errAt == atClusterWriting { + select { + case err := <-chFatal: + if !errs.Is(err, bytes.ErrTooLarge) { + t.Fatalf("Expected error: '%v', got: '%v'", bytes.ErrTooLarge, err) + } + return + case err := <-chError: + t.Fatalf("Unexpected error: '%v'", err) + case <-time.After(time.Second): + t.Fatal("Error is not emitted on write error") + } + } + if !w.WaitWrite() { + t.Fatal("Cluster is not written") + } + + time.Sleep(50 * time.Millisecond) + + if errAt == atFrameWriting { + w.setError(bytes.ErrTooLarge) + } + clearErr() + if _, err := ws[0].Write(false, 110, []byte{0x01, 0x02}); err != nil { + t.Fatalf("Failed to Write: '%v'", err) + } + if errAt == atFrameWriting { + select { + case err := <-chFatal: + if !errs.Is(err, bytes.ErrTooLarge) { + t.Fatalf("Expected error: '%v', got: '%v'", bytes.ErrTooLarge, err) + } + return + case err := <-chError: + t.Fatalf("Unexpected error: '%v'", err) + case <-time.After(time.Second): + t.Fatal("Error is not emitted on write error") + } + } + if !w.WaitWrite() { + t.Fatal("Second frame is not written") + } + + // Very old frame + clearErr() + if _, err := ws[0].Write(true, -32769, []byte{0x0A}); err != nil { + t.Fatalf("Failed to Write: '%v'", err) + } + select { + case err := <-chError: + if !errs.Is(err, ErrIgnoreOldFrame) { + t.Errorf("Expected error: '%v', got: '%v'", ErrIgnoreOldFrame, err) + } + case err := <-chFatal: + t.Fatalf("Unexpected fatal: '%v'", err) + case <-time.After(time.Second): + t.Fatal("Error is not emitted for old frame") + } + + if errAt == atClosing { + w.setError(bytes.ErrTooLarge) + } + clearErr() + ws[0].Close() + if errAt == atClosing { + select { + case err := <-chFatal: + if !errs.Is(err, bytes.ErrTooLarge) { + t.Fatalf("Expected error: '%v', got: '%v'", bytes.ErrTooLarge, err) + } + return + case err := <-chError: + t.Fatalf("Unexpected error: '%v'", err) + case <-time.After(time.Second): + t.Fatal("Error is not emitted on write error") + } + } + }) + } +} + +func TestBlockWriter_WithMaxKeyframeInterval(t *testing.T) { + buf := buffercloser.New() + + ws, err := NewSimpleBlockWriter( + buf, + []TrackDescription{{TrackNumber: 1}}, + WithEBMLHeader(nil), + WithSegmentInfo(nil), + WithMaxKeyframeInterval(1, 900*0x6FFF), + WithSeekHead(false), + ) + if err != nil { + t.Fatalf("Failed to create BlockWriter: '%v'", err) + } + if len(ws) != 1 { + t.Fatalf("Number of the returned writer must be 1, got %d", len(ws)) + } + + for _, block := range []struct { + keyframe bool + timecode int64 + b []byte + }{ + {true, 0, []byte{0x01}}, + {false, 1, []byte{0x02}}, + {false, 0x1000, []byte{0x03}}, + {true, 0x1001, []byte{0x04}}, // This will be the head of the next cluster + } { + if _, err := ws[0].Write(block.keyframe, block.timecode, block.b); err != nil { + t.Fatalf("Failed to Write: '%v'", err) + } + } + + ws[0].Close() + + expectedBytes := []byte{ + // Segment + 0x18, 0x53, 0x80, 0x67, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + // Tracks + 0x16, 0x54, 0xAE, 0x6B, 0x80, + // Cluster + 0x1F, 0x43, 0xB6, 0x75, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xE7, 0x81, 0x00, + 0xA3, 0x85, 0x81, 0x00, 0x00, 0x80, 0x01, // block 0 + 0xA3, 0x85, 0x81, 0x00, 0x01, 0x00, 0x02, // block 1 + 0xA3, 0x85, 0x81, 0x10, 0x00, 0x00, 0x03, // block 2 + // New cluster + 0x1F, 0x43, 0xB6, 0x75, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xE7, 0x82, 0x10, 0x01, + 0xAB, 0x81, 0x24, + 0xA3, 0x85, 0x81, 0x00, 0x00, 0x80, 0x04, // block 3 + // Finalization + 0x1F, 0x43, 0xB6, 0x75, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xE7, 0x82, 0x10, 0x01, + 0xAB, 0x81, 0x1A, + } + if !bytes.Equal(buf.Bytes(), expectedBytes) { + t.Errorf("Unexpected binary,\nexpected: %+v\n got: %+v", expectedBytes, buf.Bytes()) + } +} + +func TestBlockWriter_WithSeekHead(t *testing.T) { + t.Run("GenerateSeekHead", func(t *testing.T) { + buf := buffercloser.New() + + ws, err := NewSimpleBlockWriter( + buf, + []TrackDescription{{TrackNumber: 1}}, + WithEBMLHeader(nil), + WithSegmentInfo(&struct { + TimecodeScale uint64 `ebml:"TimecodeScale"` + }{TimecodeScale: 1000000}), + WithSeekHead(true), + ) + if err != nil { + t.Fatalf("Failed to create BlockWriter: '%v'", err) + } + if len(ws) != 1 { + t.Fatalf("Number of the returned writer must be 1, got %d", len(ws)) + } + + ws[0].Close() + + expectedBytes := []byte{ + // 1 2 3 4 5 6 7 8 9 10 11 12 + // Segment + 0x18, 0x53, 0x80, 0x67, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + // SeekHead + 0x11, 0x4D, 0x9B, 0x74, 0xAA, + 0x4D, 0xBB, 0x92, + 0x53, 0xAB, 0x84, 0x15, 0x49, 0xA9, 0x66, // Info + 0x53, 0xAC, 0x88, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2F, + 0x4D, 0xBB, 0x92, + 0x53, 0xAB, 0x84, 0x16, 0x54, 0xAE, 0x6B, // Tracks + 0x53, 0xAC, 0x88, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3B, + // Info, pos: 47 + 0x15, 0x49, 0xA9, 0x66, 0x87, + 0x2A, 0xD7, 0xB1, 0x83, 0x0F, 0x42, 0x40, + // Tracks, pos: 59 + 0x16, 0x54, 0xAE, 0x6B, 0x80, + // Cluster + 0x1F, 0x43, 0xB6, 0x75, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xE7, 0x81, 0x00, + } + if !bytes.Equal(buf.Bytes(), expectedBytes) { + t.Errorf("Unexpected binary,\nexpected: %+v\n got: %+v", expectedBytes, buf.Bytes()) + } + }) + t.Run("InvalidHeader", func(t *testing.T) { + buf := buffercloser.New() + + _, err := NewSimpleBlockWriter( + buf, + []TrackDescription{{TrackNumber: 1}}, + WithSegmentInfo(&struct { + Invalid uint64 `ebml:"InvalidA"` + }{}), + WithSeekHead(true), + ) + if !errs.Is(err, ebml.ErrUnknownElementName) { + t.Errorf("Expected error: '%v', got: '%v'", ebml.ErrUnknownElementName, err) + } + }) +} + +func BenchmarkBlockWriter_InitFinalize(b *testing.B) { + tracks := []TrackDescription{ + {TrackNumber: 1}, + } + + for i := 0; i < b.N; i++ { + buf := buffercloser.New() + + blockSorter, err := NewMultiTrackBlockSorter(WithMaxDelayedPackets(10), WithSortRule(BlockSorterDropOutdated)) + if err != nil { + b.Fatalf("Failed to create MultiTrackBlockSorter: %v", err) + } + + ws, err := NewSimpleBlockWriter(buf, tracks, + WithBlockInterceptor(blockSorter), + ) + if err != nil { + b.Fatalf("Failed to create BlockWriter: %v", err) + } + for _, w := range ws { + w.Close() + } + } +} + +func BenchmarkBlockWriter_SimpleBlock(b *testing.B) { + tracks := []TrackDescription{ + {TrackNumber: 1}, + } + + buf := buffercloser.New() + + blockSorter, err := NewMultiTrackBlockSorter(WithMaxDelayedPackets(10), WithSortRule(BlockSorterDropOutdated)) + if err != nil { + b.Fatalf("Failed to create MultiTrackBlockSorter: %v", err) + } + + ws, err := NewSimpleBlockWriter(buf, tracks, + WithBlockInterceptor(blockSorter), + ) + if err != nil { + b.Fatalf("Failed to create BlockWriter: %v", err) + } + + data := []byte{} + b.ResetTimer() + for i := 0; i < b.N; i++ { + for _, w := range ws { + if _, err := w.Write(true, int64(i*20), data); err != nil { + b.Fatalf("Failed to Write: %v", err) + } + } + } + b.StopTimer() + for _, w := range ws { + w.Close() + } +} diff --git a/pkg/ebml-go/mkvcore/framebuf.go b/pkg/ebml-go/mkvcore/framebuf.go new file mode 100644 index 0000000..2dd846f --- /dev/null +++ b/pkg/ebml-go/mkvcore/framebuf.go @@ -0,0 +1,53 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mkvcore + +type frameBuffer struct { + buf []*frame +} + +func (b *frameBuffer) Push(f *frame) { + b.buf = append(b.buf, f) +} +func (b *frameBuffer) Head() *frame { + if len(b.buf) == 0 { + return nil + } + return b.buf[0] +} +func (b *frameBuffer) Tail() *frame { + if len(b.buf) == 0 { + return nil + } + return b.buf[len(b.buf)-1] +} +func (b *frameBuffer) Pop() *frame { + n := len(b.buf) + if n == 0 { + return nil + } + head := b.buf[0] + b.buf[0] = nil + + if n == 1 { + b.buf = nil + } else { + b.buf = b.buf[1:] + } + return head +} +func (b *frameBuffer) Size() int { + return len(b.buf) +} diff --git a/pkg/ebml-go/mkvcore/framebuf_test.go b/pkg/ebml-go/mkvcore/framebuf_test.go new file mode 100644 index 0000000..71676e3 --- /dev/null +++ b/pkg/ebml-go/mkvcore/framebuf_test.go @@ -0,0 +1,63 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mkvcore + +import ( + "reflect" + "testing" +) + +func TestFrameBuffer(t *testing.T) { + buf := &frameBuffer{} + + if h := buf.Pop(); h != nil { + t.Errorf("Pop() must return nil if empty, expected: nil, got %v", h) + } + + if f := buf.Tail(); f != nil { + t.Errorf("Tail() must return nil if empty, expected: nil, got %v", f) + } + + if n := buf.Size(); n != 0 { + t.Errorf("Size() must return 0 at beginning, got %d", n) + } + if h := buf.Head(); h != nil { + t.Errorf("Head() must return nil at beginning, got %v", h) + } + + frames := []frame{ + {trackNumber: 2}, + {trackNumber: 3}, + } + buf.Push(&frames[0]) + buf.Push(&frames[1]) + + if n := buf.Size(); n != 2 { + t.Errorf("Size() must return 2 after pushing two frames, got %d", n) + } + if h := buf.Head(); !reflect.DeepEqual(*h, frames[0]) { + t.Errorf("Head() must return first frame, expected: %v, got %v", frames[0].trackNumber, *h) + } + if f := buf.Tail(); !reflect.DeepEqual(*f, frames[1]) { + t.Errorf("Tail() must return last frame, expected: %v, got %v", frames[1].trackNumber, *f) + } + + if h := buf.Pop(); !reflect.DeepEqual(*h, frames[0]) { + t.Errorf("Pop() must return first frame, expected: %v, got %v", frames[0].trackNumber, *h) + } + if n := buf.Size(); n != 1 { + t.Errorf("Size() must return 1 after popping one frames, got %d", n) + } +} diff --git a/pkg/ebml-go/mkvcore/interceptor.go b/pkg/ebml-go/mkvcore/interceptor.go new file mode 100644 index 0000000..ff51f94 --- /dev/null +++ b/pkg/ebml-go/mkvcore/interceptor.go @@ -0,0 +1,237 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mkvcore + +import ( + "fmt" + "io" + "sync" +) + +// BlockInterceptor is a interface of block stream muxer. +type BlockInterceptor interface { + // Intercept reads blocks of each track, filters, and writes. + Intercept(r []BlockReader, w []BlockWriter) +} + +// MustBlockInterceptor panics if creation of a BlockInterceptor fails, such as +// when the NewMultiTrackBlockSorter function fails. +func MustBlockInterceptor(interceptor BlockInterceptor, err error) BlockInterceptor { + if err != nil { + panic(err) + } + return interceptor +} + +type filterWriter struct { + trackNumber uint64 + ch chan *frame +} + +type filterReader struct { + ch chan *frame +} + +func (w *filterWriter) Write(keyframe bool, timestamp int64, b []byte) (int, error) { + w.ch <- &frame{ + trackNumber: w.trackNumber, + keyframe: keyframe, + timestamp: timestamp, + b: b, + } + return len(b), nil +} + +func (r *filterReader) Read() ([]byte, bool, int64, error) { + frame, ok := <-r.ch + if !ok { + return nil, false, 0, io.EOF + } + return frame.b, frame.keyframe, frame.timestamp, nil +} + +func (r *filterReader) close() { + close(r.ch) +} + +// BlockSorterRule is a type of BlockSorter behaviour for outdated frame. +type BlockSorterRule int + +// List of BlockSorterRules. +const ( + BlockSorterDropOutdated BlockSorterRule = iota + BlockSorterWriteOutdated +) + +// MultiTrackBlockSorterOption configures a MultiTrackBlockSorterOptions. +type MultiTrackBlockSorterOption func(*MultiTrackBlockSorterOptions) error + +// MultiTrackBlockSorterOptions stores options for BlockWriter. +type MultiTrackBlockSorterOptions struct { + maxDelayedPackets int + rule BlockSorterRule + maxTimescaleDelay int64 +} + +// WithMaxDelayedPackets set the maximum number of packets that may be delayed +// within each track. +func WithMaxDelayedPackets(maxDelayedPackets int) MultiTrackBlockSorterOption { + return func(o *MultiTrackBlockSorterOptions) error { + o.maxDelayedPackets = maxDelayedPackets + return nil + } +} + +// WithSortRule set the sort rule to apply to how packet ordering should be +// treated within the webm container. +func WithSortRule(rule BlockSorterRule) MultiTrackBlockSorterOption { + return func(o *MultiTrackBlockSorterOptions) error { + o.rule = rule + return nil + } +} + +// WithMaxTimescaleDelay set the maximum allowed delay between tracks for a +// given timescale. +func WithMaxTimescaleDelay(maxTimescaleDelay int64) MultiTrackBlockSorterOption { + return func(o *MultiTrackBlockSorterOptions) error { + o.maxTimescaleDelay = maxTimescaleDelay + return nil + } +} + +// NewMultiTrackBlockSorter creates BlockInterceptor, which sorts blocks on +// multiple tracks by timestamp. Either WithMaxDelayedPackets or +// WithMaxTimescaleDelay must be specified. If both are specified, then the +// first rule that is satisfied causes the packets to get written (thus a +// backlog of a max packets or max time scale will cause any older packets than +// the one satisfying the rule to be discarded). The index of TrackEntry sorts +// blocks with the same timestamp. Place the audio track before the video track +// to meet WebM Interceptor Guidelines. +func NewMultiTrackBlockSorter(opts ...MultiTrackBlockSorterOption) (BlockInterceptor, error) { + applyOptions := []MultiTrackBlockSorterOption{ + WithMaxDelayedPackets(0), + WithSortRule(BlockSorterDropOutdated), + WithMaxTimescaleDelay(0), + } + applyOptions = append(applyOptions, opts...) + + options := &MultiTrackBlockSorterOptions{} + for _, o := range applyOptions { + if err := o(options); err != nil { + return nil, err + } + } + + if options.maxDelayedPackets == 0 && options.maxTimescaleDelay == 0 { + return nil, fmt.Errorf("must specify either WithMaxDelayedPackets(...) or WithMaxTimescaleDelay(...) with a non-0 value") + } + + return &multiTrackBlockSorter{options: *options}, nil +} + +type multiTrackBlockSorter struct { + options MultiTrackBlockSorterOptions +} + +func (s *multiTrackBlockSorter) Intercept(r []BlockReader, w []BlockWriter) { + var wg sync.WaitGroup + wg.Add(len(r)) + + ch := make(chan *frame) + for i, r := range r { + go func(i int, r BlockReader) { + for { + var err error + f := &frame{trackNumber: uint64(i)} + if f.b, f.keyframe, f.timestamp, err = r.Read(); err != nil { + wg.Done() + return + } + ch <- f + } + }(i, r) + } + + closed := make(chan struct{}) + go func() { + wg.Wait() + close(closed) + }() + + var tDone int64 + buf := make([]*frameBuffer, len(r)) + for i := range buf { + buf[i] = &frameBuffer{} + } + + flush := func(all bool) { + nChReq := 1 + if !all { + nChReq = len(r) + } + for { + var largestTimestampDelta int64 + var tOldest int64 + var tNewest int64 + var nCh, nMax int + var bOldest *frameBuffer + var bNewest *frameBuffer + for _, b := range buf { + if n := b.Size(); n > 0 { + nCh++ + if f := b.Head(); f.timestamp < tOldest || bOldest == nil { + tOldest = f.timestamp + bOldest = b + } + if f := b.Tail(); f.timestamp > tNewest || bNewest == nil { + tNewest = f.timestamp + bNewest = b + + tDiff := tNewest - tOldest + if tDiff > largestTimestampDelta { + largestTimestampDelta = tDiff + } + } + if n > nMax { + nMax = n + } + } + } + if nCh >= nChReq || + (nMax > s.options.maxDelayedPackets && s.options.maxDelayedPackets != 0) || + (largestTimestampDelta > s.options.maxTimescaleDelay && s.options.maxTimescaleDelay != 0) { + fOldest := bOldest.Pop() + _, _ = w[fOldest.trackNumber].Write(fOldest.keyframe, fOldest.timestamp, fOldest.b) + tDone = fOldest.timestamp + } else { + break + } + } + } + + for { + select { + case d := <-ch: + if d.timestamp >= tDone || s.options.rule == BlockSorterWriteOutdated { + buf[d.trackNumber].Push(d) + flush(false) + } + case <-closed: + flush(true) + return + } + } +} diff --git a/pkg/ebml-go/mkvcore/interceptor_test.go b/pkg/ebml-go/mkvcore/interceptor_test.go new file mode 100644 index 0000000..92ff80d --- /dev/null +++ b/pkg/ebml-go/mkvcore/interceptor_test.go @@ -0,0 +1,330 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mkvcore + +import ( + "errors" + "reflect" + "sync" + "testing" + "time" +) + +func TestMultiTrackBlockSorterMaxPackets(t *testing.T) { + for name, c := range map[string]struct { + rule BlockSorterRule + expected []frame + }{ + "DropOutdated": { + BlockSorterDropOutdated, + []frame{ + {1, false, 9, []byte{3}}, + {0, false, 10, []byte{1}}, + {0, false, 11, []byte{2}}, + {0, false, 16, []byte{4}}, + {0, false, 17, []byte{5}}, + {0, false, 18, []byte{6}}, + {1, false, 18, []byte{8}}, + }, + }, + "WriteOutdated": { + BlockSorterWriteOutdated, + []frame{ + {1, false, 9, []byte{3}}, + {0, false, 10, []byte{1}}, + {0, false, 11, []byte{2}}, + {0, false, 16, []byte{4}}, + {1, false, 15, []byte{7}}, + {0, false, 17, []byte{5}}, + {0, false, 18, []byte{6}}, + {1, false, 18, []byte{8}}, + }, + }, + } { + t.Run(name, func(t *testing.T) { + wg := sync.WaitGroup{} + + f, err := NewMultiTrackBlockSorter(WithMaxDelayedPackets(2), WithSortRule(c.rule)) + if err != nil { + t.Errorf("Failed to create MultiTrackBlockSorter: %v", err) + } + + chOut := make(chan *frame) + ch := []chan *frame{ + make(chan *frame), + make(chan *frame), + } + + w := []BlockWriter{ + &filterWriter{0, chOut}, + &filterWriter{1, chOut}, + } + r := []BlockReader{ + &filterReader{ch[0]}, + &filterReader{ch[1]}, + } + + var frames []frame + wg.Add(1) + go func() { + for f := range chOut { + frames = append(frames, *f) + } + wg.Done() + }() + + go func() { + ch[0] <- &frame{0, false, 10, []byte{1}} + ch[0] <- &frame{0, false, 11, []byte{2}} + time.Sleep(time.Millisecond) + ch[1] <- &frame{1, false, 9, []byte{3}} + time.Sleep(time.Millisecond) + ch[0] <- &frame{0, false, 16, []byte{4}} + ch[0] <- &frame{0, false, 17, []byte{5}} + ch[0] <- &frame{0, false, 18, []byte{6}} + time.Sleep(time.Millisecond) + ch[1] <- &frame{1, false, 15, []byte{7}} // drop due to maxDelay=2 + ch[1] <- &frame{1, false, 18, []byte{8}} + close(ch[0]) + close(ch[1]) + }() + + f.Intercept(r, w) + + close(chOut) + wg.Wait() + + if !reflect.DeepEqual(c.expected, frames) { + t.Errorf("Unexpected sort result, \nexpected: %v, \n got: %v", c.expected, frames) + } + }) + } +} + +func TestMultiTrackBlockSorterTimescale(t *testing.T) { + for name, c := range map[string]struct { + rule BlockSorterRule + expected []frame + }{ + "DropOutdated": { + BlockSorterDropOutdated, + []frame{ + {0, false, 100, []byte{1}}, + {0, false, 110, []byte{2}}, + {1, false, 150, []byte{3}}, + {0, false, 160, []byte{4}}, + {0, false, 170, []byte{5}}, + {0, false, 200, []byte{6}}, + {1, false, 210, []byte{8}}, + }, + }, + "WriteOutdated": { + BlockSorterWriteOutdated, + []frame{ + {0, false, 100, []byte{1}}, + {0, false, 110, []byte{2}}, + {1, false, 150, []byte{3}}, + {1, false, 90, []byte{7}}, + {0, false, 160, []byte{4}}, + {0, false, 170, []byte{5}}, + {0, false, 200, []byte{6}}, + {1, false, 210, []byte{8}}, + }, + }, + } { + t.Run(name, func(t *testing.T) { + wg := sync.WaitGroup{} + + f, err := NewMultiTrackBlockSorter(WithMaxTimescaleDelay(100), WithSortRule(c.rule)) + if err != nil { + t.Errorf("Failed to create MultiTrackBlockSorter: %v", err) + } + + chOut := make(chan *frame) + ch := []chan *frame{ + make(chan *frame), + make(chan *frame), + } + + w := []BlockWriter{ + &filterWriter{0, chOut}, + &filterWriter{1, chOut}, + } + r := []BlockReader{ + &filterReader{ch[0]}, + &filterReader{ch[1]}, + } + + var frames []frame + wg.Add(1) + go func() { + for f := range chOut { + frames = append(frames, *f) + } + wg.Done() + }() + + go func() { + ch[0] <- &frame{0, false, 100, []byte{1}} + ch[0] <- &frame{0, false, 110, []byte{2}} + time.Sleep(time.Millisecond) + ch[1] <- &frame{1, false, 150, []byte{3}} + time.Sleep(time.Millisecond) + ch[0] <- &frame{0, false, 160, []byte{4}} + ch[0] <- &frame{0, false, 170, []byte{5}} + ch[0] <- &frame{0, false, 200, []byte{6}} + time.Sleep(time.Millisecond) + ch[1] <- &frame{1, false, 90, []byte{7}} // maybe dropped due to WithMaxTimescaleDelay=100 + ch[1] <- &frame{1, false, 210, []byte{8}} + close(ch[0]) + close(ch[1]) + }() + + f.Intercept(r, w) + + close(chOut) + wg.Wait() + + if !reflect.DeepEqual(c.expected, frames) { + t.Errorf("Unexpected sort result, \nexpected: %v, \n got: %v", c.expected, frames) + } + }) + } +} + +func BenchmarkMultiTrackBlockSorter(b *testing.B) { + f, err := NewMultiTrackBlockSorter(WithMaxDelayedPackets(2), WithSortRule(BlockSorterDropOutdated)) + if err != nil { + b.Errorf("Failed to create MultiTrackBlockSorter: %v", err) + } + + chOut := make(chan *frame) + ch := []chan *frame{ + make(chan *frame), + make(chan *frame), + } + + w := []BlockWriter{ + &filterWriter{0, chOut}, + &filterWriter{1, chOut}, + } + r := []BlockReader{ + &filterReader{ch[0]}, + &filterReader{ch[1]}, + } + + go func() { + for range chOut { + } + }() + + go func() { + for i := 0; i < b.N; i++ { + ch[0] <- &frame{0, false, int64(i), []byte{1, 2, 3, 4}} + ch[1] <- &frame{1, false, int64(i) + 5, []byte{2, 3, 4, 5}} + } + close(ch[0]) + close(ch[1]) + }() + + b.ResetTimer() + f.Intercept(r, w) + + close(chOut) +} + +func TestMultiTrackBlockSorter_FailingOptions(t *testing.T) { + errDummy := errors.New("an error") + + cases := map[string]struct { + opts []MultiTrackBlockSorterOption + err error + }{ + "SingleOptionPackets": { + opts: []MultiTrackBlockSorterOption{ + WithMaxDelayedPackets(2), + }, + err: nil, + }, + "SingleOptionTimeScape": { + opts: []MultiTrackBlockSorterOption{ + WithMaxTimescaleDelay(2), + }, + err: nil, + }, + "MultiOptionPacketsAndTimeScale": { + opts: []MultiTrackBlockSorterOption{ + WithMaxDelayedPackets(2), + WithMaxTimescaleDelay(100), + WithSortRule(BlockSorterDropOutdated), + }, + err: nil, + }, + "FailingOption": { + opts: []MultiTrackBlockSorterOption{ + WithSortRule(BlockSorterDropOutdated), + }, + err: errDummy, + }, + "ErroredOption": { + opts: []MultiTrackBlockSorterOption{ + func(*MultiTrackBlockSorterOptions) error { + return errDummy + }, + }, + err: errDummy, + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + _, err := NewMultiTrackBlockSorter(c.opts...) + if c.err == nil && err != nil { + t.Errorf("Expecting no error but got: '%v'", err) + } + if c.err != nil && err == nil { + t.Errorf("Expected error but didn't get one: '%v'", name) + } + }) + } +} + +type dummyInterceptor struct{} + +func (dummyInterceptor) Intercept([]BlockReader, []BlockWriter) { + panic("unimplemented") +} + +func TestMustBlockInterceptor(t *testing.T) { + t.Run("OK", func(t *testing.T) { + i := &dummyInterceptor{} + ret := MustBlockInterceptor(i, nil) + if ret != i { + t.Error("MustBlockInterceptor must return the interceptor on success") + } + }) + t.Run("Error", func(t *testing.T) { + i := &dummyInterceptor{} + err := errors.New("dummy error") + + defer func() { + if r := recover(); r == nil { + t.Error("MustBlockInterceptor must panic on failure") + } + }() + + _ = MustBlockInterceptor(i, err) + }) +} diff --git a/pkg/ebml-go/mkvcore/interface.go b/pkg/ebml-go/mkvcore/interface.go new file mode 100644 index 0000000..bd13172 --- /dev/null +++ b/pkg/ebml-go/mkvcore/interface.go @@ -0,0 +1,58 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mkvcore + +// BlockWriter is a Matroska block writer interface. +type BlockWriter interface { + // Write a block to the connected Matroska writer. + // timestamp is in millisecond. + Write(keyframe bool, timestamp int64, b []byte) (int, error) +} + +// BlockReader is a Matroska block reader interface. +type BlockReader interface { + // Read a block from the connected Matroska reader. + Read() (b []byte, keyframe bool, timestamp int64, err error) +} + +// BlockCloser is a Matroska closer interface. +type BlockCloser interface { + // Close the stream frame writer. + // Output Matroska will be closed after closing all FrameWriter. + Close() error +} + +// BlockWriteCloser groups Writer and Closer. +type BlockWriteCloser interface { + BlockWriter + BlockCloser +} + +// BlockReadCloser groups Reader and Closer. +type BlockReadCloser interface { + BlockReader + BlockCloser +} + +// TrackEntryGetter is a interface to get TrackEntry. +type TrackEntryGetter interface { + TrackEntry() TrackEntry +} + +// BlockReadCloserWithTrackEntry groups BlockReadCloser and TrackEntryGetter. +type BlockReadCloserWithTrackEntry interface { + BlockReadCloser + TrackEntryGetter +} diff --git a/pkg/ebml-go/mkvcore/mkvcore.go b/pkg/ebml-go/mkvcore/mkvcore.go new file mode 100644 index 0000000..37484da --- /dev/null +++ b/pkg/ebml-go/mkvcore/mkvcore.go @@ -0,0 +1,18 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package mkvcore provides the core functionality of Matroska/WebM multimedia writer. +// +// The package implements block data writer for EBML based multi-track media container. +package mkvcore diff --git a/pkg/ebml-go/mkvcore/option.go b/pkg/ebml-go/mkvcore/option.go new file mode 100644 index 0000000..a7ba816 --- /dev/null +++ b/pkg/ebml-go/mkvcore/option.go @@ -0,0 +1,165 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mkvcore + +import ( + "errors" + + "github.com/at-wat/ebml-go" +) + +// ErrInvalidTrackNumber means that a track number is invalid. The track number must be larger than 0. +var ErrInvalidTrackNumber = errors.New("invalid track number") + +// BlockWriterOption configures a BlockWriterOptions. +type BlockWriterOption interface { + ApplyToBlockWriterOptions(opts *BlockWriterOptions) error +} + +// BlockReaderOption configures a BlockReaderOptions. +type BlockReaderOption interface { + ApplyToBlockReaderOptions(opts *BlockReaderOptions) error +} + +// BlockReadWriterOptionFn configures a BlockReadWriterOptions. +type BlockReadWriterOptionFn func(*BlockReadWriterOptions) error + +// ApplyToBlockWriterOptions implements BlockWriterOption. +func (o BlockReadWriterOptionFn) ApplyToBlockWriterOptions(opts *BlockWriterOptions) error { + return o(&opts.BlockReadWriterOptions) +} + +// ApplyToBlockReaderOptions implements BlockReaderOption. +func (o BlockReadWriterOptionFn) ApplyToBlockReaderOptions(opts *BlockReaderOptions) error { + return o(&opts.BlockReadWriterOptions) +} + +// BlockReadWriterOptions stores options for BlockWriter and BlockReader. +type BlockReadWriterOptions struct { + onError func(error) + onFatal func(error) +} + +// WithOnErrorHandler registers marshal error handler. +func WithOnErrorHandler(handler func(error)) BlockReadWriterOptionFn { + return func(o *BlockReadWriterOptions) error { + o.onError = handler + return nil + } +} + +// WithOnFatalHandler registers marshal error handler. +func WithOnFatalHandler(handler func(error)) BlockReadWriterOptionFn { + return func(o *BlockReadWriterOptions) error { + o.onFatal = handler + return nil + } +} + +// BlockWriterOptionFn configures a BlockWriterOptions. +type BlockWriterOptionFn func(*BlockWriterOptions) error + +// ApplyToBlockWriterOptions implements BlockWriterOption. +func (o BlockWriterOptionFn) ApplyToBlockWriterOptions(opts *BlockWriterOptions) error { + return o(opts) +} + +// BlockWriterOptions stores options for BlockWriter. +type BlockWriterOptions struct { + BlockReadWriterOptions + ebmlHeader interface{} + segmentInfo interface{} + seekHead bool + marshalOpts []ebml.MarshalOption + interceptor BlockInterceptor + mainTrackNumber uint64 + maxKeyframeInterval int64 +} + +// WithEBMLHeader sets EBML header. +func WithEBMLHeader(h interface{}) BlockWriterOptionFn { + return func(o *BlockWriterOptions) error { + o.ebmlHeader = h + return nil + } +} + +// WithSegmentInfo sets Segment.Info. +func WithSegmentInfo(i interface{}) BlockWriterOptionFn { + return func(o *BlockWriterOptions) error { + o.segmentInfo = i + return nil + } +} + +// WithSeekHead enables SeekHead calculation +func WithSeekHead(enable bool) BlockWriterOptionFn { + return func(o *BlockWriterOptions) error { + o.seekHead = enable + return nil + } +} + +// WithMarshalOptions passes ebml.MarshalOption to ebml.Marshal. +func WithMarshalOptions(opts ...ebml.MarshalOption) BlockWriterOptionFn { + return func(o *BlockWriterOptions) error { + o.marshalOpts = opts + return nil + } +} + +// WithBlockInterceptor registers BlockInterceptor. +func WithBlockInterceptor(interceptor BlockInterceptor) BlockWriterOptionFn { + return func(o *BlockWriterOptions) error { + o.interceptor = interceptor + return nil + } +} + +// WithMaxKeyframeInterval sets maximum keyframe interval of the main (video) track. +// Using this option starts the cluster with a key frame if possible. +// interval must be given in the scale of timecode. +func WithMaxKeyframeInterval(mainTrackNumber uint64, interval int64) BlockWriterOptionFn { + return func(o *BlockWriterOptions) error { + if mainTrackNumber == 0 { + return ErrInvalidTrackNumber + } + o.mainTrackNumber = mainTrackNumber + o.maxKeyframeInterval = interval + return nil + } +} + +// BlockReaderOptionFn configures a BlockReaderOptions. +type BlockReaderOptionFn func(*BlockReaderOptions) error + +// ApplyToBlockReaderOptions implements BlockReaderOption. +func (o BlockReaderOptionFn) ApplyToBlockReaderOptions(opts *BlockReaderOptions) error { + return o(opts) +} + +// BlockReaderOptions stores options for BlockReader. +type BlockReaderOptions struct { + BlockReadWriterOptions + unmarshalOpts []ebml.UnmarshalOption +} + +// WithUnmarshalOptions passes ebml.UnmarshalOption to ebml.Unmarshal. +func WithUnmarshalOptions(opts ...ebml.UnmarshalOption) BlockReaderOptionFn { + return func(o *BlockReaderOptions) error { + o.unmarshalOpts = opts + return nil + } +} diff --git a/pkg/ebml-go/mkvcore/seekhead.go b/pkg/ebml-go/mkvcore/seekhead.go new file mode 100644 index 0000000..5ff3921 --- /dev/null +++ b/pkg/ebml-go/mkvcore/seekhead.go @@ -0,0 +1,61 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mkvcore + +import ( + "bytes" + + "github.com/at-wat/ebml-go" +) + +func setSeekHead(header *flexHeader, opts ...ebml.MarshalOption) error { + infoPos := new(uint64) + tracksPos := new(uint64) + header.Segment.SeekHead = &seekHeadFixed{} + if header.Segment.Info != nil { + header.Segment.SeekHead.Seek = append(header.Segment.SeekHead.Seek, seekFixed{ + SeekID: ebml.ElementInfo.Bytes(), + SeekPosition: infoPos, + }) + } + header.Segment.SeekHead.Seek = append(header.Segment.SeekHead.Seek, seekFixed{ + SeekID: ebml.ElementTracks.Bytes(), + SeekPosition: tracksPos, + }) + + var segmentPos uint64 + hook := func(e *ebml.Element) { + switch e.Name { + case "SeekHead": + // SeekHead position is the top of the Segment contents. + // Origin of the segment position is here. + segmentPos = e.Position + case "Info": + *infoPos = e.Position - segmentPos + case "Tracks": + *tracksPos = e.Position - segmentPos + } + } + + optsWithHook := append([]ebml.MarshalOption{}, opts...) + optsWithHook = append(optsWithHook, ebml.WithElementWriteHooks(hook)) + + var buf bytes.Buffer + if err := ebml.Marshal(header, &buf, optsWithHook...); err != nil { + return err + } + + return nil +} diff --git a/pkg/ebml-go/mkvcore/sizedwriter.go b/pkg/ebml-go/mkvcore/sizedwriter.go new file mode 100644 index 0000000..ae8272e --- /dev/null +++ b/pkg/ebml-go/mkvcore/sizedwriter.go @@ -0,0 +1,41 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mkvcore + +import ( + "io" +) + +type writerWithSizeCount struct { + size int + w io.WriteCloser +} + +func (w *writerWithSizeCount) Write(b []byte) (int, error) { + w.size += len(b) + return w.w.Write(b) +} + +func (w *writerWithSizeCount) Clear() { + w.size = 0 +} + +func (w *writerWithSizeCount) Close() error { + return w.w.Close() +} + +func (w *writerWithSizeCount) Size() int { + return w.size +} diff --git a/pkg/ebml-go/mkvcore/sizedwriter_test.go b/pkg/ebml-go/mkvcore/sizedwriter_test.go new file mode 100644 index 0000000..283d847 --- /dev/null +++ b/pkg/ebml-go/mkvcore/sizedwriter_test.go @@ -0,0 +1,50 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mkvcore + +import ( + "testing" + + "github.com/at-wat/ebml-go/internal/buffercloser" +) + +func TestWriterWithSizeCount(t *testing.T) { + buf := buffercloser.New() + w := &writerWithSizeCount{w: buf} + + if n, err := w.Write([]byte{0x01, 0x02}); err != nil { + t.Fatalf("Failed to Write: '%v'", err) + } else if n != 2 { + t.Errorf("Expected return value of writerWithSizeCount.Write: 2, got: %d", n) + } + if n := w.Size(); n != 2 { + t.Errorf("Expected return value of writerWithSizeCount.Size(): 2, got: %d", n) + } + + w.Clear() + + if n := w.Size(); n != 0 { + t.Errorf("Expected return value of writerWithSizeCount.Size(): 0, got: %d", n) + } + + if err := w.Close(); err != nil { + t.Errorf("writerWithSizeCount.Close() doesn't propagate base io.WriteCloser.Close() return value") + } + select { + case <-buf.Closed(): + default: + t.Errorf("Base io.WriteCloser is not closed by writerWithSizeCount.Close()") + } +} diff --git a/pkg/ebml-go/mkvcore/struct.go b/pkg/ebml-go/mkvcore/struct.go new file mode 100644 index 0000000..4726314 --- /dev/null +++ b/pkg/ebml-go/mkvcore/struct.go @@ -0,0 +1,76 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mkvcore + +import ( + "github.com/at-wat/ebml-go" +) + +type simpleBlockGroup struct { + Block []ebml.Block `ebml:"Block"` + ReferencePriority uint64 `ebml:"ReferencePriority"` +} + +type simpleBlockCluster struct { + Timecode uint64 `ebml:"Timecode"` + PrevSize uint64 `ebml:"PrevSize,omitempty"` + SimpleBlock []ebml.Block `ebml:"SimpleBlock"` + BlockGroup []simpleBlockGroup `ebml:"BlockGroup,omitempty"` +} + +type seekFixed struct { + SeekID []byte `ebml:"SeekID"` + SeekPosition *uint64 `ebml:"SeekPosition,size=8"` +} + +type seekHeadFixed struct { + Seek []seekFixed `ebml:"Seek"` +} + +type flexTracks struct { + TrackEntry []interface{} `ebml:"TrackEntry"` +} + +type flexSegment struct { + SeekHead *seekHeadFixed `ebml:"SeekHead,omitempty"` + Info interface{} `ebml:"Info"` + Tracks flexTracks `ebml:"Tracks"` + Cluster []simpleBlockCluster `ebml:"Cluster,size=unknown"` +} + +type flexHeader struct { + Header interface{} `ebml:"EBML"` + Segment flexSegment `ebml:"Segment,size=unknown"` +} + +// TrackEntry is a TrackEntry struct with all mandatory elements and commonly used elements. +type TrackEntry struct { + TrackNumber uint64 + TrackUID uint64 + TrackType uint8 + FlagEnabled uint8 + FlagDefault uint8 + FlagForced uint8 + FlagLacing uint8 + MinCache uint64 + DefaultDuration uint64 + MaxBlockAdditionID uint64 + Name string + Language string + LanguageIETF string + CodecID string + CodecDecodeAll uint8 + SeekPreRoll uint64 +} diff --git a/pkg/ebml-go/reader.go b/pkg/ebml-go/reader.go new file mode 100644 index 0000000..7b6a348 --- /dev/null +++ b/pkg/ebml-go/reader.go @@ -0,0 +1,79 @@ +// Copyright 2021 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ebml + +import ( + "bytes" + "io" +) + +type rollbackReader interface { + Set(io.Reader) + Get() io.Reader + Read([]byte) (int, error) + Reset() + RollbackTo(int) +} + +type rollbackReaderImpl struct { + io.Reader + buf []byte +} + +func (r *rollbackReaderImpl) Set(v io.Reader) { + r.Reader = v +} + +func (r *rollbackReaderImpl) Get() io.Reader { + return r.Reader +} + +func (r *rollbackReaderImpl) Read(b []byte) (int, error) { + n, err := r.Reader.Read(b) + r.buf = append(r.buf, b[:n]...) + return n, err +} + +func (r *rollbackReaderImpl) Reset() { + r.buf = r.buf[0:0] +} + +func (r *rollbackReaderImpl) RollbackTo(i int) { + buf := r.buf + r.Reader = io.MultiReader( + bytes.NewReader(buf[i:]), + r.Reader, + ) + r.buf = nil +} + +type rollbackReaderNop struct { + io.Reader +} + +func (r *rollbackReaderNop) Set(v io.Reader) { + r.Reader = v +} + +func (r *rollbackReaderNop) Get() io.Reader { + return r.Reader +} + +func (*rollbackReaderNop) Reset() { +} + +func (*rollbackReaderNop) RollbackTo(i int) { + panic("can't rollback nop rollback reader") +} diff --git a/pkg/ebml-go/reader_test.go b/pkg/ebml-go/reader_test.go new file mode 100644 index 0000000..248ae31 --- /dev/null +++ b/pkg/ebml-go/reader_test.go @@ -0,0 +1,103 @@ +// Copyright 2021 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ebml + +import ( + "bytes" + "io" + "testing" +) + +func TestRollbackReader(t *testing.T) { + r := &rollbackReaderImpl{ + Reader: bytes.NewReader([]byte{0, 1, 2, 3, 4, 5, 6, 7}), + } + + b := make([]byte, 3) + n, err := io.ReadFull(r, b) + if err != nil { + t.Fatal(err) + } + if n != 3 { + t.Fatalf("Expected to read 3 bytes, got %d bytes", n) + } + if !bytes.Equal([]byte{0, 1, 2}, b) { + t.Fatalf("Unexpected read result: %v", b) + } + + r.Reset() + + n, err = io.ReadFull(r, b) + if err != nil { + t.Fatal(err) + } + if n != 3 { + t.Fatalf("Expected to read 3 bytes, got %d bytes", n) + } + if !bytes.Equal([]byte{3, 4, 5}, b) { + t.Fatalf("Unexpected read result: %v", b) + } + + r.RollbackTo(1) + + n, err = io.ReadFull(r, b) + if err != nil { + t.Fatal(err) + } + if n != 3 { + t.Fatalf("Expected to read 3 bytes, got %d bytes", n) + } + if !bytes.Equal([]byte{4, 5, 6}, b) { + t.Fatalf("Unexpected read result: %v", b) + } +} + +func TestRollbackReaderNop(t *testing.T) { + r := &rollbackReaderNop{ + Reader: bytes.NewReader([]byte{0, 1, 2, 3, 4, 5, 6, 7}), + } + + b := make([]byte, 3) + n, err := r.Read(b) + if err != nil { + t.Fatal(err) + } + if n != 3 { + t.Fatalf("Expected to read 3 bytes, got %d bytes", n) + } + if !bytes.Equal([]byte{0, 1, 2}, b) { + t.Fatalf("Unexpected read result: %v", b) + } + + r.Reset() + + n, err = r.Read(b) + if err != nil { + t.Fatal(err) + } + if n != 3 { + t.Fatalf("Expected to read 3 bytes, got %d bytes", n) + } + if !bytes.Equal([]byte{3, 4, 5}, b) { + t.Fatalf("Unexpected read result: %v", b) + } + + defer func() { + if err := recover(); err == nil { + t.Error("Expected panic") + } + }() + r.RollbackTo(1) +} diff --git a/pkg/ebml-go/tag.go b/pkg/ebml-go/tag.go new file mode 100644 index 0000000..734f5b9 --- /dev/null +++ b/pkg/ebml-go/tag.go @@ -0,0 +1,83 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ebml + +import ( + "errors" + "os" + "strconv" + "strings" +) + +type structTag struct { + name string + size uint64 + omitEmpty bool + stop bool +} + +// ErrEmptyTag means that a tag string has empty item. +var ErrEmptyTag = errors.New("empty tag in tag string") + +// ErrInvalidTag means that an invaild tag is specified. +var ErrInvalidTag = errors.New("invalid tag in tag string") + +func parseTag(rawtag string) (*structTag, error) { + tag := &structTag{} + + ts := strings.Split(rawtag, ",") + + for i, t := range ts { + if i == 0 { + tag.name = t + continue + } + kv := strings.SplitN(t, "=", 2) + if len(kv) == 1 { + switch kv[0] { + case "": + return nil, ErrEmptyTag + case "omitempty": + tag.omitEmpty = true + case "inf": + os.Stderr.WriteString("Deprecated: \"inf\" tag is replaced by \"size=unknown\"\n") + tag.size = SizeUnknown + case "stop": + tag.stop = true + default: + return nil, wrapErrorf(ErrInvalidTag, "parsing \"%s\"", t) + } + continue + } + + switch kv[0] { + case "": + return nil, ErrEmptyTag + case "size": + if kv[1] == "unknown" { + tag.size = SizeUnknown + } else { + s, err := strconv.Atoi(kv[1]) + if err != nil { + return nil, wrapErrorf(err, "parsing \"%s\"", t) + } + tag.size = uint64(s) + } + default: + return nil, wrapErrorf(ErrInvalidTag, "parsing \"%s\"", t) + } + } + return tag, nil +} diff --git a/pkg/ebml-go/tag_test.go b/pkg/ebml-go/tag_test.go new file mode 100644 index 0000000..db0fa75 --- /dev/null +++ b/pkg/ebml-go/tag_test.go @@ -0,0 +1,97 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ebml + +import ( + "reflect" + "strconv" + "testing" + + "github.com/at-wat/ebml-go/internal/errs" +) + +func TestParseTag(t *testing.T) { + cases := map[string]struct { + input string + expected *structTag + err error + }{ + "Empty": { + "", + &structTag{}, nil, + }, + "Name": { + "Name123", + &structTag{name: "Name123"}, nil, + }, + "OmitEmpty": { + "Name123,omitempty", + &structTag{name: "Name123", omitEmpty: true}, nil, + }, + "OmitEmptyWithDefaultName": { + ",omitempty", + &structTag{omitEmpty: true}, nil, + }, + "Size": { + "Name123,size=45", + &structTag{name: "Name123", size: 45}, nil, + }, + "UnknownSize": { + "Name123,size=unknown", + &structTag{name: "Name123", size: SizeUnknown}, nil, + }, + "UnknownSizeDeprecated": { + "Name123,inf", + &structTag{name: "Name123", size: SizeUnknown}, nil, + }, + "InvalidSize": { + "Name123,size=a", + nil, strconv.ErrSyntax, + }, + "InvalidTag": { + "Name,invalidtag", + nil, ErrInvalidTag, + }, + "InvalidTagWithValue": { + "Name,invalidtag=1", + nil, ErrInvalidTag, + }, + "EmptyTag": { + "Name,", + nil, ErrEmptyTag, + }, + "EmptyTagWithValue": { + "Name,=45", + nil, ErrEmptyTag, + }, + "TwoEmptyTags": { + "Name,,", + nil, ErrEmptyTag, + }, + } + for n, c := range cases { + t.Run(n, func(t *testing.T) { + tag, err := parseTag(c.input) + if !errs.Is(err, c.err) { + t.Errorf("Expected error: '%v', got: '%v'", c.err, err) + } + if (c.expected == nil) != (tag == nil) { + t.Errorf("Expected output nil-ness: %v, got: %v", c.expected == nil, tag == nil) + } else if tag != nil && !reflect.DeepEqual(*c.expected, *tag) { + t.Errorf("Expected output: %v, got: %v", *c.expected, *tag) + } + }) + } +} diff --git a/pkg/ebml-go/testutils_test.go b/pkg/ebml-go/testutils_test.go new file mode 100644 index 0000000..5707d76 --- /dev/null +++ b/pkg/ebml-go/testutils_test.go @@ -0,0 +1,35 @@ +package ebml + +import ( + "bytes" + "io" +) + +type limitedDummyWriter struct { + n int + limit int +} + +func (s *limitedDummyWriter) Write(b []byte) (int, error) { + s.n += len(b) + if s.n > s.limit { + return len(b) - (s.n - s.limit), bytes.ErrTooLarge + } + return len(b), nil +} + +type delayedBrokenReader struct { + b []byte + n int + limit int +} + +func (s *delayedBrokenReader) Read(b []byte) (int, error) { + p := s.n + s.n += len(b) + if s.n > s.limit { + return len(b) - (s.n - s.limit), io.ErrClosedPipe + } + copy(b, s.b[p:p+len(b)]) + return len(b), nil +} diff --git a/pkg/ebml-go/unlacer.go b/pkg/ebml-go/unlacer.go new file mode 100644 index 0000000..e793d57 --- /dev/null +++ b/pkg/ebml-go/unlacer.go @@ -0,0 +1,173 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ebml + +import ( + "errors" + "io" +) + +// ErrFixedLaceUndivisible means that a length of a fixed lacing data is undivisible. +var ErrFixedLaceUndivisible = errors.New("undivisible fixed lace") + +// Unlacer is the interface to read laced frames in Block. +type Unlacer interface { + Read() ([]byte, error) +} + +type unlacer struct { + r io.Reader + i int + size []int +} + +func (u *unlacer) Read() ([]byte, error) { + if u.i >= len(u.size) { + return nil, io.EOF + } + n := u.size[u.i] + u.i++ + + b := make([]byte, n) + _, err := io.ReadFull(u.r, b) + return b, err +} + +// NewNoUnlacer creates pass-through Unlacer for not laced data. +func NewNoUnlacer(r io.Reader, n int64) (Unlacer, error) { + return &unlacer{r: r, size: []int{int(n)}}, nil +} + +// NewXiphUnlacer creates Unlacer for Xiph laced data. +func NewXiphUnlacer(r io.Reader, n int64) (Unlacer, error) { + var nFrame int + var b [1]byte + switch _, err := r.Read(b[:]); err { + case nil: + nFrame = int(b[0]) + 1 + case io.EOF: + return nil, io.ErrUnexpectedEOF + default: + return nil, err + } + n-- + + ul := &unlacer{ + r: r, + size: make([]int, nFrame), + } + for i := 0; i < nFrame-1; i++ { + for { + switch _, err := ul.r.Read(b[:]); err { + case nil: + case io.EOF: + return nil, io.ErrUnexpectedEOF + default: + return nil, err + } + n-- + ul.size[i] += int(b[0]) + if b[0] != 0xFF { + ul.size[nFrame-1] -= ul.size[i] + break + } + } + } + ul.size[nFrame-1] += int(n) + if ul.size[nFrame-1] <= 0 { + return nil, io.ErrUnexpectedEOF + } + + return ul, nil +} + +// NewFixedUnlacer creates Unlacer for Fixed laced data. +func NewFixedUnlacer(r io.Reader, n int64) (Unlacer, error) { + var nFrame int + var b [1]byte + switch _, err := r.Read(b[:]); err { + case nil: + nFrame = int(b[0]) + 1 + case io.EOF: + return nil, io.ErrUnexpectedEOF + default: + return nil, err + } + + ul := &unlacer{ + r: r, + size: make([]int, nFrame), + } + ul.size[0] = (int(n) - 1) / nFrame + for i := 1; i < nFrame; i++ { + ul.size[i] = ul.size[0] + } + if ul.size[0]*nFrame+1 != int(n) { + return nil, wrapErrorf( + ErrFixedLaceUndivisible, "unlacing %d bytes of %d frames", n-1, nFrame, + ) + } + return ul, nil +} + +// NewEBMLUnlacer creates Unlacer for EBML laced data. +func NewEBMLUnlacer(r io.Reader, n int64) (Unlacer, error) { + var nFrame int + var b [1]byte + switch _, err := r.Read(b[:]); err { + case nil: + nFrame = int(b[0]) + 1 + case io.EOF: + return nil, io.ErrUnexpectedEOF + default: + return nil, err + } + n-- + + vd := &valueDecoder{} + + ul := &unlacer{ + r: r, + size: make([]int, nFrame), + } + un64, nRead, err := vd.readVUInt(ul.r) + if err != nil { + return nil, err + } + n64 := int64(un64) + n -= int64(nRead) + ul.size[0] = int(n64) + ul.size[nFrame-1] -= int(n64) + + for i := 1; i < nFrame-1; i++ { + n64Diff, nRead, err := vd.readVInt(ul.r) + n64 += int64(n64Diff) + if err != nil { + return nil, err + } + if n64 <= 0 { + return nil, io.ErrUnexpectedEOF + } + n -= int64(nRead) + ul.size[i] = int(n64) + ul.size[nFrame-1] -= int(n64) + } + ul.size[nFrame-1] += int(n) + if ul.size[nFrame-1] <= 0 { + return nil, io.ErrUnexpectedEOF + } + + return ul, nil +} diff --git a/pkg/ebml-go/unlacer_test.go b/pkg/ebml-go/unlacer_test.go new file mode 100644 index 0000000..cbcaf86 --- /dev/null +++ b/pkg/ebml-go/unlacer_test.go @@ -0,0 +1,208 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ebml + +import ( + "bytes" + "io" + "testing" + + "github.com/at-wat/ebml-go/internal/errs" +) + +func TestUnlacer(t *testing.T) { + cases := map[string]struct { + newUnlacer func(io.Reader, int64) (Unlacer, error) + header []byte + frames [][]byte + err error + }{ + "Xiph": { + newUnlacer: NewXiphUnlacer, + header: []byte{ + 0x02, + 0xFF, 0x01, // 256 bytes + 0x10, // 16 bytes + }, + frames: [][]byte{ + bytes.Repeat([]byte{0xAA}, 256), + bytes.Repeat([]byte{0xCC}, 16), + bytes.Repeat([]byte{0x55}, 8), + }, + err: nil, + }, + "XiphEmpty": { + newUnlacer: NewXiphUnlacer, + header: []byte{}, + frames: [][]byte{}, + err: io.ErrUnexpectedEOF, + }, + "XiphShort": { + newUnlacer: NewXiphUnlacer, + header: []byte{ + 0x02, 0xFF, + }, + frames: [][]byte{}, + err: io.ErrUnexpectedEOF, + }, + "XiphMissingFrame": { + newUnlacer: NewXiphUnlacer, + header: []byte{ + 0x02, + 0x02, + 0x01, + }, + frames: [][]byte{{0x00, 0x01}}, + err: io.ErrUnexpectedEOF, + }, + "XiphMissingLastFrame": { + newUnlacer: NewXiphUnlacer, + header: []byte{ + 0x02, + 0x02, + 0x01, + }, + frames: [][]byte{{0x00, 0x01}, {0x02}}, + err: io.ErrUnexpectedEOF, + }, + "Fixed": { + newUnlacer: NewFixedUnlacer, + header: []byte{ + 0x02, + }, + frames: [][]byte{ + bytes.Repeat([]byte{0xAA}, 16), + bytes.Repeat([]byte{0xCC}, 16), + bytes.Repeat([]byte{0x55}, 16), + }, + err: nil, + }, + "FixedEmpty": { + newUnlacer: NewFixedUnlacer, + header: []byte{}, + frames: [][]byte{}, + err: io.ErrUnexpectedEOF, + }, + "FixedUndivisible": { + newUnlacer: NewFixedUnlacer, + header: []byte{ + 0x02, + }, + frames: [][]byte{ + bytes.Repeat([]byte{0xAA}, 16), + bytes.Repeat([]byte{0xCC}, 16), + bytes.Repeat([]byte{0x55}, 15), + }, + err: ErrFixedLaceUndivisible, + }, + "EBML": { + newUnlacer: NewEBMLUnlacer, + header: []byte{ + 0x02, + 0x43, 0x20, // 800 bytes + 0x5E, 0xD3, // 500 bytes + }, + frames: [][]byte{ + bytes.Repeat([]byte{0xAA}, 800), + bytes.Repeat([]byte{0xCC}, 500), + bytes.Repeat([]byte{0x55}, 100), + }, + err: nil, + }, + "EBMLEmpty": { + newUnlacer: NewEBMLUnlacer, + header: []byte{}, + frames: [][]byte{}, + err: io.ErrUnexpectedEOF, + }, + "EBMLInvalidSize": { + newUnlacer: NewEBMLUnlacer, + header: []byte{ + 0x02, + 0x41, + }, + frames: [][]byte{}, + err: io.ErrUnexpectedEOF, + }, + "EBMLInvalidSize2": { + newUnlacer: NewEBMLUnlacer, + header: []byte{ + 0x03, + 0x81, + }, + frames: [][]byte{}, + err: io.ErrUnexpectedEOF, + }, + "EBMLNegativeSize": { + newUnlacer: NewEBMLUnlacer, + header: []byte{ + 0x03, + 0x81, // 1 byte + 0x80, // -62 bytes + }, + frames: [][]byte{}, + err: io.ErrUnexpectedEOF, + }, + "EBMLMissingFrame": { + newUnlacer: NewEBMLUnlacer, + header: []byte{ + 0x02, + 0x82, // 2 bytes + 0x9F, // 2 bytes + }, + frames: [][]byte{{0x00, 0x01}}, + err: io.ErrUnexpectedEOF, + }, + "EBMLMissingLastFrame": { + newUnlacer: NewEBMLUnlacer, + header: []byte{ + 0x02, + 0x82, // 2 bytes + 0x9E, // 1 byte + }, + frames: [][]byte{{0x00, 0x01}, {0x02}}, + err: io.ErrUnexpectedEOF, + }, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + b := append([]byte{}, c.header...) + for _, f := range c.frames { + b = append(b, f...) + } + + ul, err := c.newUnlacer(bytes.NewReader(b), int64(len(b))) + if !errs.Is(err, c.err) { + t.Fatalf("Expected error: '%v', got: '%v'", c.err, err) + } + if err != nil { + return + } + + for _, f := range c.frames { + b, err := ul.Read() + if err != nil { + t.Fatalf("Unexpected error: '%v'", err) + } + if !bytes.Equal(f, b) { + t.Errorf("Unexpected data, \nexpected: %v, \n got: %v", f, b) + } + } + if _, err := ul.Read(); err != io.EOF { + t.Fatalf("Unexpected error: '%v'", err) + } + }) + } +} diff --git a/pkg/ebml-go/unmarshal.go b/pkg/ebml-go/unmarshal.go new file mode 100644 index 0000000..07c4c20 --- /dev/null +++ b/pkg/ebml-go/unmarshal.go @@ -0,0 +1,318 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ebml + +import ( + "bytes" + "errors" + "io" + "reflect" +) + +// ErrUnknownElement means that a decoded element is not known. +var ErrUnknownElement = errors.New("unknown element") + +// ErrIndefiniteType means that a marshal/unmarshal destination type is not valid. +var ErrIndefiniteType = errors.New("marshal/unmarshal to indefinite type") + +// ErrIncompatibleType means that an element is not convertible to a corresponding struct field. +var ErrIncompatibleType = errors.New("marshal/unmarshal to incompatible type") + +// ErrInvalidElementSize means that an element has inconsistent size. e.g. element size is larger than its parent element size. +var ErrInvalidElementSize = errors.New("invalid element size") + +// ErrReadStopped is returned if unmarshaler finished to read element which has stop tag. +var ErrReadStopped = errors.New("read stopped") + +// Unmarshal EBML stream. +func Unmarshal(r io.Reader, val interface{}, opts ...UnmarshalOption) error { + options := &UnmarshalOptions{} + for _, o := range opts { + if err := o(options); err != nil { + return err + } + } + + vo := reflect.ValueOf(val) + if !vo.IsValid() { + return wrapErrorf(ErrIndefiniteType, "unmarshalling to %T", val) + } + if vo.Kind() != reflect.Ptr { + return wrapErrorf(ErrIncompatibleType, "unmarshalling to %T", val) + } + + vd := &valueDecoder{} + + voe := vo.Elem() + for { + if _, err := vd.readElement(r, SizeUnknown, voe, 0, 0, nil, options); err != nil { + if err == io.EOF { + return nil + } + return err + } + } +} + +func (vd *valueDecoder) readElement(r0 io.Reader, n int64, vo reflect.Value, depth int, pos uint64, parent *Element, options *UnmarshalOptions) (io.Reader, error) { + pos0 := pos + var r rollbackReader + if options.ignoreUnknown { + r = &rollbackReaderImpl{} + } else { + r = &rollbackReaderNop{} + } + if n != SizeUnknown { + r.Set(io.LimitReader(r0, n)) + } else { + r.Set(r0) + } + + var mapOut bool + type fieldDef struct { + v reflect.Value + stop bool + } + fieldMap := make(map[ElementType]fieldDef) + switch vo.Kind() { + case reflect.Struct: + for i := 0; i < vo.NumField(); i++ { + f := fieldDef{ + v: vo.Field(i), + } + var name string + if n, ok := vo.Type().Field(i).Tag.Lookup("ebml"); ok { + t, err := parseTag(n) + if err != nil { + return nil, err + } + name = t.name + f.stop = t.stop + } + if name == "" { + name = vo.Type().Field(i).Name + } + t, err := ElementTypeFromString(name) + if err != nil { + return nil, err + } + fieldMap[t] = f + } + case reflect.Map: + mapOut = true + } + + for { + r.Reset() + + var headerSize uint64 + e, nb, err := vd.readVUInt(r) + headerSize += uint64(nb) + if err != nil { + if nb == 0 && err == io.ErrUnexpectedEOF { + return nil, io.EOF + } + if options.ignoreUnknown { + return nil, nil + } + return nil, err + } + v, ok := revTable[uint32(e)] + if !ok { + if options.ignoreUnknown { + r.RollbackTo(1) + pos++ + continue + } + return nil, wrapErrorf(ErrUnknownElement, "unmarshalling element 0x%x", e) + } + + size, nb, err := vd.readDataSize(r) + headerSize += uint64(nb) + + if n != SizeUnknown && pos+headerSize+size > pos0+uint64(n) { + err = ErrInvalidElementSize + } + + if err != nil { + if options.ignoreUnknown { + r.RollbackTo(1) + pos++ + continue + } + return nil, err + } + + var vnext reflect.Value + var stopHere bool + if vn, ok := fieldMap[v.e]; ok { + if !mapOut { + vnext = vn.v + } + stopHere = vn.stop + } + + var chanSend reflect.Value + var elem *Element + if len(options.hooks) > 0 && vnext.IsValid() { + elem = &Element{ + Name: v.e.String(), + Type: v.e, + Position: pos, + Size: size, + Parent: parent, + } + } + if vnext.Kind() == reflect.Chan { + chanSend = vnext + vnext = reflect.New(vnext.Type().Elem()).Elem() + } + + switch v.t { + case DataTypeMaster: + if v.top && depth > 1 { + b := bytes.Join([][]byte{table[v.e].b, encodeDataSize(size, uint64(nb))}, []byte{}) + return bytes.NewBuffer(b), io.EOF + } + var vn reflect.Value + if mapOut { + vnext = reflect.ValueOf(make(map[string]interface{})) + vn = vnext + } else { + if vnext.IsValid() && vnext.CanSet() { + switch vnext.Kind() { + case reflect.Ptr: + vnext.Set(reflect.New(vnext.Type().Elem())) + vn = vnext.Elem() + case reflect.Slice: + vnext.Set(reflect.Append(vnext, reflect.New(vnext.Type().Elem()).Elem())) + vn = vnext.Index(vnext.Len() - 1) + default: + vn = vnext + } + } + } + if elem != nil { + elem.Value = vn.Interface() + } + r0, err := vd.readElement(r, int64(size), vn, depth+1, pos+headerSize, elem, options) + if err != nil && err != io.EOF { + return r0, err + } + if r0 != nil { + r.Set(io.MultiReader(r0, r.Get())) + } + default: + val, err := vd.decode(v.t, r, size) + if err != nil { + if options.ignoreUnknown { + r.RollbackTo(1) + pos++ + continue + } + return nil, err + } + vr := reflect.ValueOf(val) + if mapOut { + vnext = vr + } else { + if vnext.IsValid() && vnext.CanSet() { + switch { + case vr.Type() == vnext.Type(): + vnext.Set(vr) + case isConvertible(vr.Type(), vnext.Type()): + vnext.Set(vr.Convert(vnext.Type())) + case vnext.Kind() == reflect.Slice: + t := vnext.Type().Elem() + switch { + case vr.Type() == t: + vnext.Set(reflect.Append(vnext, vr)) + case isConvertible(vr.Type(), t): + vnext.Set(reflect.Append(vnext, vr.Convert(t))) + default: + return nil, wrapErrorf( + ErrIncompatibleType, "unmarshalling %s to %s", vnext.Type(), vr.Type(), + ) + } + default: + return nil, wrapErrorf( + ErrIncompatibleType, "unmarshalling %s to %s", vnext.Type(), vr.Type(), + ) + } + } + } + if elem != nil { + elem.Value = vr.Interface() + } + } + if mapOut { + t := vo.Type() + if vo.IsNil() && t.Kind() == reflect.Map { + vo.Set(reflect.MakeMap(t)) + } + key := reflect.ValueOf(v.e.String()) + if e := vo.MapIndex(key); e.IsValid() { + switch { + case e.Elem().Kind() == reflect.Slice && v.t != DataTypeBinary: + vnext = reflect.Append(e.Elem(), vnext) + default: + vnext = reflect.ValueOf([]interface{}{ + e.Elem().Interface(), + vnext.Interface()}, + ) + } + } + vo.SetMapIndex(key, vnext) + } + if chanSend.IsValid() { + chanSend.Send(vnext) + } + if elem != nil { + for _, hook := range options.hooks { + hook(elem) + } + } + + pos += headerSize + size + if stopHere { + return nil, ErrReadStopped + } + } +} + +// UnmarshalOption configures a UnmarshalOptions struct. +type UnmarshalOption func(*UnmarshalOptions) error + +// UnmarshalOptions stores options for unmarshalling. +type UnmarshalOptions struct { + hooks []func(elem *Element) + ignoreUnknown bool +} + +// WithElementReadHooks returns an UnmarshalOption which registers element hooks. +func WithElementReadHooks(hooks ...func(*Element)) UnmarshalOption { + return func(opts *UnmarshalOptions) error { + opts.hooks = hooks + return nil + } +} + +// WithIgnoreUnknown returns an UnmarshalOption which makes Unmarshal ignoring unknown element with static length. +func WithIgnoreUnknown(ignore bool) UnmarshalOption { + return func(opts *UnmarshalOptions) error { + opts.ignoreUnknown = ignore + return nil + } +} diff --git a/pkg/ebml-go/unmarshal_test.go b/pkg/ebml-go/unmarshal_test.go new file mode 100644 index 0000000..f561ca7 --- /dev/null +++ b/pkg/ebml-go/unmarshal_test.go @@ -0,0 +1,688 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ebml + +import ( + "bytes" + "errors" + "fmt" + "io" + "reflect" + "testing" + "time" + + "github.com/at-wat/ebml-go/internal/errs" +) + +func ExampleUnmarshal() { + TestBinary := []byte{ + 0x1a, 0x45, 0xdf, 0xa3, // EBML + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, // 0x10 + 0x42, 0x82, 0x85, 0x77, 0x65, 0x62, 0x6d, 0x00, // EBMLDocType = webm + 0x42, 0x87, 0x81, 0x02, // DocTypeVersion = 2 + 0x42, 0x85, 0x81, 0x02, // DocTypeReadVersion = 2 + } + type TestEBML struct { + Header struct { + DocType string `ebml:"EBMLDocType"` + DocTypeVersion uint64 `ebml:"EBMLDocTypeVersion"` + DocTypeReadVersion uint64 `ebml:"EBMLDocTypeReadVersion"` + } `ebml:"EBML"` + } + + r := bytes.NewReader(TestBinary) + + var ret TestEBML + if err := Unmarshal(r, &ret); err != nil { + fmt.Printf("error: %v\n", err) + } + fmt.Println(ret) + + // Output: {{webm 2 2}} +} + +func TestUnmarshal_MultipleUnknownSize(t *testing.T) { + b := []byte{ + 0x18, 0x53, 0x80, 0x67, // Segment + 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x1F, 0x43, 0xB6, 0x75, // Cluster + 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x42, 0x87, 0x81, 0x01, + 0x1F, 0x43, 0xB6, 0x75, // Cluster + 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x42, 0x87, 0x81, 0x02, + } + type Cluster struct { + DocTypeVersion uint64 `ebml:"EBMLDocTypeVersion"` + } + type Segment struct { + Cluster []Cluster `ebml:"Cluster"` + } + type TestEBML struct { + Segment Segment `ebml:"Segment"` + } + expected := TestEBML{ + Segment: Segment{ + Cluster: []Cluster{{0x01}, {0x02}}, + }, + } + + var ret TestEBML + if err := Unmarshal(bytes.NewReader(b), &ret); err != nil { + t.Fatalf("Unexpected error: '%v'\n", err) + } + if !reflect.DeepEqual(expected, ret) { + t.Errorf("Expected result: %v, got: %v", expected, ret) + } +} + +func TestUnmarshal_Convert(t *testing.T) { + cases := map[string]struct { + b []byte + expected interface{} + }{ + "UInt64ToUInt64": { + []byte{0x42, 0x87, 0x81, 0x02}, + struct { + DocTypeVersion uint64 `ebml:"EBMLDocTypeVersion"` + }{2}, + }, + "UInt64ToUInt32": { + []byte{0x42, 0x87, 0x81, 0x02}, + struct { + DocTypeVersion uint32 `ebml:"EBMLDocTypeVersion"` + }{2}, + }, + "UInt64ToUInt16": { + []byte{0x42, 0x87, 0x81, 0x02}, + struct { + DocTypeVersion uint16 `ebml:"EBMLDocTypeVersion"` + }{2}, + }, + "UInt64ToUInt8": { + []byte{0x42, 0x87, 0x81, 0x02}, + struct { + DocTypeVersion uint8 `ebml:"EBMLDocTypeVersion"` + }{2}, + }, + "UInt64ToUInt": { + []byte{0x42, 0x87, 0x81, 0x02}, + struct { + DocTypeVersion uint `ebml:"EBMLDocTypeVersion"` + }{2}, + }, + "Int64ToInt64": { + []byte{0xFB, 0x81, 0xFF}, + struct { + ReferenceBlock int64 `ebml:"ReferenceBlock"` + }{-1}, + }, + "Int64ToInt32": { + []byte{0xFB, 0x81, 0xFF}, + struct { + ReferenceBlock int32 `ebml:"ReferenceBlock"` + }{-1}, + }, + "Int64ToInt16": { + []byte{0xFB, 0x81, 0xFF}, + struct { + ReferenceBlock int16 `ebml:"ReferenceBlock"` + }{-1}, + }, + "Int64ToInt8": { + []byte{0xFB, 0x81, 0xFF}, + struct { + ReferenceBlock int8 `ebml:"ReferenceBlock"` + }{-1}, + }, + "Int64ToInt": { + []byte{0xFB, 0x81, 0xFF}, + struct { + ReferenceBlock int `ebml:"ReferenceBlock"` + }{-1}, + }, + "Float64ToFloat64": { + []byte{0x44, 0x89, 0x84, 0x00, 0x00, 0x00, 0x00}, + struct { + Duration float64 `ebml:"Duration"` + }{0.0}, + }, + "Float64ToFloat32": { + []byte{0x44, 0x89, 0x84, 0x00, 0x00, 0x00, 0x00}, + struct { + Duration float32 `ebml:"Duration"` + }{0.0}, + }, + "UInt64ToUInt64Slice": { + []byte{0x42, 0x87, 0x81, 0x02}, + struct { + DocTypeVersion []uint64 `ebml:"EBMLDocTypeVersion"` + }{[]uint64{2}}, + }, + "UInt64ToUInt32Slice": { + []byte{0x42, 0x87, 0x81, 0x02}, + struct { + DocTypeVersion []uint32 `ebml:"EBMLDocTypeVersion"` + }{[]uint32{2}}, + }, + "Int64ToInt32Slice": { + []byte{0xFB, 0x81, 0xFF}, + struct { + ReferenceBlock []int32 `ebml:"ReferenceBlock"` + }{[]int32{-1}}, + }, + "Float64ToFloat32Slice": { + []byte{0x44, 0x89, 0x84, 0x00, 0x00, 0x00, 0x00}, + struct { + Duration []float32 `ebml:"Duration"` + }{[]float32{0.0}}, + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + ret := reflect.New(reflect.ValueOf(c.expected).Type()) + if err := Unmarshal(bytes.NewReader(c.b), ret.Interface()); err != nil { + t.Fatalf("Unexpected error: '%v'\n", err) + } + + if !reflect.DeepEqual(c.expected, ret.Elem().Interface()) { + t.Errorf("Expected convert result: %v, got %v", + c.expected, ret.Elem().Interface()) + } + }) + } +} + +func TestUnmarshal_OptionError(t *testing.T) { + errExpected := errors.New("an error") + err := Unmarshal(&bytes.Buffer{}, &struct{}{}, + func(*UnmarshalOptions) error { + return errExpected + }, + ) + if err != errExpected { + t.Errorf("Expected error against failing UnmarshalOption: '%v', got: '%v'", errExpected, err) + } +} + +func TestUnmarshal_WithElementReadHooks(t *testing.T) { + TestBinary := []byte{ + 0x18, 0x53, 0x80, 0x67, 0xa6, // Segment + 0x1c, 0x53, 0xbb, 0x6b, 0x80, // Cues (empty) + 0x16, 0x54, 0xae, 0x6b, 0x9c, // Tracks + 0xae, 0x8c, // TrackEntry[0] + 0x53, 0x6e, 0x86, 0x56, 0x69, 0x64, 0x65, 0x6f, 0x00, // Name=Video + 0xd7, 0x81, 0x01, // TrackNumber=1 + 0xae, 0x8c, // TrackEntry[1] + 0x53, 0x6e, 0x86, 0x41, 0x75, 0x64, 0x69, 0x6f, 0x00, // Name=Audio + 0xd7, 0x81, 0x02, // TrackNumber=2 + } + + type TestEBML struct { + Segment struct { + Tracks struct { + TrackEntry []struct { + Name string `ebml:"Name,omitempty"` + TrackNumber uint64 `ebml:"TrackNumber"` + } `ebml:"TrackEntry"` + } `ebml:"Tracks"` + } `ebml:"Segment"` + } + + r := bytes.NewReader(TestBinary) + + var ret TestEBML + m := make(map[string][]*Element) + hook := withElementMap(m) + if err := Unmarshal(r, &ret, WithElementReadHooks(hook)); err != nil { + t.Errorf("Unexpected error: '%v'", err) + } + + // Verify positions of elements + expected := map[string][]uint64{ + "Segment": {0}, + "Segment.Tracks": {10}, + "Segment.Tracks.TrackEntry": {15, 29}, + "Segment.Tracks.TrackEntry.Name": {17, 31}, + "Segment.Tracks.TrackEntry.TrackNumber": {26, 40}, + } + posMap := elementPositionMap(m) + if !reflect.DeepEqual(expected, posMap) { + t.Errorf("Unexpected read hook positions, \nexpected: %v, \n got: %v", expected, posMap) + } + checkTarget := "Segment.Tracks.TrackEntry.Name" + switch { + case len(m[checkTarget]) != 2: + t.Fatalf("%s read hook should be called twice, but called %d times", + checkTarget, len(m[checkTarget])) + case m[checkTarget][0].Type != ElementName: + t.Fatalf("ElementType of %s should be %s, got %s", + checkTarget, ElementName, m[checkTarget][0].Type) + } + switch v, ok := m[checkTarget][0].Value.(string); { + case !ok: + t.Errorf("Invalid type of data: %T", v) + case v != "Video": + t.Errorf("The value should be Video, got %s", v) + } +} + +func TestUnmarshal_Chan(t *testing.T) { + TestBinary := []byte{ + 0x18, 0x53, 0x80, 0x67, 0x8f, // Segment + 0x16, 0x54, 0xae, 0x6b, 0x8a, // Tracks + 0xae, 0x83, // TrackEntry[0] + 0xd7, 0x81, 0x01, // TrackNumber=1 + 0xae, 0x83, // TrackEntry[0] + 0xd7, 0x81, 0x02, // TrackNumber=2 + } + type TestEBML struct { + Segment struct { + Tracks struct { + TrackEntry struct { + TrackNumber chan uint64 `ebml:"TrackNumber"` + } `ebml:"TrackEntry"` + } `ebml:"Tracks"` + } `ebml:"Segment"` + } + + var ret TestEBML + ch := make(chan uint64, 100) + ret.Segment.Tracks.TrackEntry.TrackNumber = ch + + done := make(chan struct{}) + go func() { + select { + case <-time.After(5 * time.Second): + panic("test timeout") + case <-done: + } + }() + if err := Unmarshal(bytes.NewReader(TestBinary), &ret); err != nil { + t.Errorf("Unexpected error: '%v'", err) + } + close(done) + if len(ch) != 2 { + t.Fatalf("Element chan should be sent twice, but sent %d times", len(ch)) + } + if v := <-ch; v != 1 { + t.Errorf("First value should be 1, got %d", v) + } + if v := <-ch; v != 2 { + t.Errorf("Second value should be 2, got %d", v) + } +} + +func TestUnmarshal_Tag(t *testing.T) { + var tagged struct { + DocCustomNamedType string `ebml:"EBMLDocType"` + } + var untagged struct { + EBMLDocType string + } + + b := []byte{0x42, 0x82, 0x85, 0x68, 0x6F, 0x67, 0x65, 0x00} + + if err := Unmarshal(bytes.NewBuffer(b), &tagged); err != nil { + t.Fatalf("Unexpected error: '%v'", err) + } + if err := Unmarshal(bytes.NewBuffer(b), &untagged); err != nil { + t.Fatalf("Unexpected error: '%v'", err) + } + + if tagged.DocCustomNamedType != untagged.EBMLDocType { + t.Errorf("Unmarshal result to tagged and and untagged struct must be same, tagged: %v, untagged: %v", tagged, untagged) + } +} + +func TestUnmarshal_Map(t *testing.T) { + b := []byte{ + 0x1A, 0x45, 0xDF, 0xA3, 0x8C, + 0x42, 0x82, 0x85, 0x68, 0x6F, 0x67, 0x65, 0x00, + 0xEC, 0x82, 0x00, 0x00, + 0x18, 0x53, 0x80, 0x67, 0xFF, + 0x1F, 0x43, 0xB6, 0x75, 0x80, + 0x1F, 0x43, 0xB6, 0x75, 0x80, + 0x1F, 0x43, 0xB6, 0x75, 0x80, + } + expected := map[string]interface{}{ + "EBML": map[string]interface{}{ + "EBMLDocType": "hoge", + "Void": []uint8{0, 0}, + }, + "Segment": map[string]interface{}{ + "Cluster": []interface{}{ + map[string]interface{}{}, + map[string]interface{}{}, + map[string]interface{}{}, + }, + }, + } + + t.Run("AllocatedMap", func(t *testing.T) { + ret := make(map[string]interface{}) + if err := Unmarshal(bytes.NewBuffer(b), &ret); err != nil { + t.Fatalf("Unexpected error: '%v'", err) + } + + if !reflect.DeepEqual(expected, ret) { + t.Errorf("Unmarshal to map differs from expected:\n%#+v\ngot:\n%#+v", expected, ret) + } + }) + + t.Run("NilMap", func(t *testing.T) { + var ret map[string]interface{} + if err := Unmarshal(bytes.NewBuffer(b), &ret); err != nil { + t.Fatalf("Unexpected error: '%v'", err) + } + + if !reflect.DeepEqual(expected, ret) { + t.Errorf("Unmarshal to map differs from expected:\n%#+v\ngot:\n%#+v", expected, ret) + } + }) +} + +func TestUnmarshal_IgnoreUnknown(t *testing.T) { + b := []byte{ + 0x1A, 0x45, 0xDF, 0xA3, 0x8A, + 0x42, 0x82, 0x85, 0x68, 0x6F, 0x67, 0x65, 0x00, + 0x81, 0x81, // 0x81 is not defined in Matroska v4 + 0x18, 0x53, 0x80, 0x67, 0xFF, + 0x1F, 0x43, 0xB6, 0x75, 0x80, + 0x1F, 0x43, 0xB6, 0x75, 0x80, + } + expected := map[string]interface{}{ + "EBML": map[string]interface{}{ + "EBMLDocType": "hoge", + }, + "Segment": map[string]interface{}{ + "Cluster": []interface{}{ + map[string]interface{}{}, + map[string]interface{}{}, + }, + }, + } + + ret := make(map[string]interface{}) + if err := Unmarshal(bytes.NewBuffer(b), &ret, WithIgnoreUnknown(true)); err != nil { + t.Fatalf("Unexpected error: '%v'", err) + } + + if !reflect.DeepEqual(expected, ret) { + t.Errorf("Unmarshal with IgnoreUnknown differs from expected:\n%#+v\ngot:\n%#+v", expected, ret) + } +} + +func TestUnmarshal_Error(t *testing.T) { + type TestEBML struct { + Header struct { + } `ebml:"EBML"` + } + t.Run("NilValue", func(t *testing.T) { + if err := Unmarshal(bytes.NewBuffer([]byte{}), nil); !errs.Is(err, ErrIndefiniteType) { + t.Errorf("Expected error: '%v', got: '%v'", ErrIndefiniteType, err) + } + }) + t.Run("NonPtr", func(t *testing.T) { + if err := Unmarshal(bytes.NewBuffer([]byte{}), struct{}{}); !errs.Is(err, ErrIncompatibleType) { + t.Errorf("Expected error: '%v', got: '%v'", ErrIncompatibleType, err) + } + }) + t.Run("UnknownElementName", func(t *testing.T) { + input := &struct { + Header struct { + } `ebml:"Unknown"` + }{} + if err := Unmarshal(bytes.NewBuffer([]byte{}), input); !errs.Is(err, ErrUnknownElementName) { + t.Errorf("Expected error: '%v', got: '%v'", ErrUnknownElementName, err) + } + }) + t.Run("InvalidTag", func(t *testing.T) { + input := &struct { + Header struct { + } `ebml:"EBML,ivalid"` + }{} + if err := Unmarshal(bytes.NewBuffer([]byte{}), input); !errs.Is(err, ErrInvalidTag) { + t.Errorf("Expected error: '%v', got: '%v'", ErrInvalidTag, err) + } + }) + t.Run("UnknownElement", func(t *testing.T) { + input := &TestEBML{} + b := []byte{0x81} + if err := Unmarshal(bytes.NewBuffer(b), input); !errs.Is(err, ErrUnknownElement) { + t.Errorf("Expected error: '%v', got: '%v'", ErrUnknownElement, err) + } + }) + t.Run("NonStaticUnknownElementWithIgnoreUnknown", func(t *testing.T) { + input := &TestEBML{} + b := []byte{0x81, 0xFF} + if err := Unmarshal( + bytes.NewBuffer(b), input, WithIgnoreUnknown(true), + ); err != nil { + t.Errorf("Unexpected error: '%v'", err) + } + }) + t.Run("ShortUnknownElementWithIgnoreUnknown", func(t *testing.T) { + input := &TestEBML{} + b := []byte{0x81, 0x85, 0x00} + if err := Unmarshal( + bytes.NewBuffer(b), input, WithIgnoreUnknown(true), + ); err != nil { + t.Errorf("Unexpected error: '%v'", err) + } + }) + t.Run("Short", func(t *testing.T) { + TestBinaries := map[string][]byte{ + "ElementID": {0x1a, 0x45, 0xdf}, + "DataSize": {0x42, 0x86, 0x40}, + "UInt(0)": {0x42, 0x86, 0x84}, + "UInt": {0x42, 0x86, 0x84, 0x00}, + "Float(0)": {0x44, 0x89, 0x84}, + "Float": {0x44, 0x89, 0x84, 0x00}, + "String(0)": {0x42, 0x82, 0x84}, + "String": {0x42, 0x82, 0x84, 0x00}, + } + for name, b := range TestBinaries { + t.Run(name, func(t *testing.T) { + var val TestEBML + if err := Unmarshal(bytes.NewBuffer(b), &val); !errs.Is(err, io.ErrUnexpectedEOF) { + t.Errorf("Expected error: '%v', got: '%v'", io.ErrUnexpectedEOF, err) + } + }) + } + }) + t.Run("ErrorPropagation", func(t *testing.T) { + TestBinaries := map[string][]byte{ + "UInt": {0x42, 0x86, 0x84, 0x00, 0x00, 0x00, 0x00}, + "Float": {0x44, 0x89, 0x84, 0x00, 0x00, 0x00, 0x00}, + "String": {0x42, 0x82, 0x84, 0x00, 0x00, 0x00, 0x00}, + "Block": {0xA3, 0x85, 0x81, 0x00, 0x00, 0x00, 0x00}, + "BlockXiph": {0xA3, 0x88, 0x81, 0x00, 0x00, 0x02, 0x01, 0x01, 0x00, 0x00}, + "BlockFixed": {0xA3, 0x87, 0x81, 0x00, 0x00, 0x04, 0x01, 0x00, 0x00}, + "BlockEBML": {0xA3, 0x88, 0x98, 0x00, 0x00, 0x06, 0x01, 0x81, 0x00, 0x00}, + } + for name, b := range TestBinaries { + t.Run(name, func(t *testing.T) { + for i := 1; i < len(b)-1; i++ { + var val TestEBML + r := &delayedBrokenReader{b: b, limit: i} + if err := Unmarshal(r, &val); !errs.Is(err, io.ErrClosedPipe) { + t.Errorf("Error is not propagated from Reader, limit: %d, expected: '%v', got: '%v'", i, io.ErrClosedPipe, err) + } + } + }) + } + }) + t.Run("Incompatible", func(t *testing.T) { + cases := map[string]struct { + b []byte + ret interface{} + err error + }{ + "UInt64ToInt64": { + b: []byte{0x42, 0x87, 0x81, 0x02}, + ret: &struct { + DocTypeVersion int64 `ebml:"EBMLDocTypeVersion"` + }{}, + err: ErrIncompatibleType, + }, + "Int64ToUInt64": { + b: []byte{0xFB, 0x81, 0xFF}, + ret: &struct { + ReferenceBlock uint64 `ebml:"ReferenceBlock"` + }{}, + err: ErrIncompatibleType, + }, + "Float64ToInt64": { + b: []byte{0x44, 0x89, 0x84, 0x00, 0x00, 0x00, 0x00}, + ret: &struct { + Duration int64 `ebml:"Duration"` + }{}, + err: ErrIncompatibleType, + }, + "StringToInt64": { + b: []byte{0x42, 0x82, 0x85, 0x77, 0x65, 0x62, 0x6d, 0x00}, + ret: &struct { + EBMLDocType int64 `ebml:"EBMLDocType"` + }{}, + err: ErrIncompatibleType, + }, + "UInt64ToInt64Slice": { + b: []byte{0x42, 0x87, 0x81, 0x02}, + ret: &struct { + DocTypeVersion []int64 `ebml:"EBMLDocTypeVersion"` + }{}, + err: ErrIncompatibleType, + }, + "Int64ToUInt64Slice": { + b: []byte{0xFB, 0x81, 0xFF}, + ret: &struct { + ReferenceBlock []uint64 `ebml:"ReferenceBlock"` + }{}, + err: ErrIncompatibleType, + }, + "Float64ToInt64Slice": { + b: []byte{0x44, 0x89, 0x84, 0x00, 0x00, 0x00, 0x00}, + ret: &struct { + Duration []int64 `ebml:"Duration"` + }{}, + err: ErrIncompatibleType, + }, + "StringToInt64Slice": { + b: []byte{0x42, 0x82, 0x85, 0x77, 0x65, 0x62, 0x6d, 0x00}, + ret: &struct { + EBMLDocType []int64 `ebml:"EBMLDocType"` + }{}, + err: ErrIncompatibleType, + }, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + if err := Unmarshal(bytes.NewBuffer(c.b), c.ret); !errs.Is(err, c.err) { + t.Errorf("Expected error: '%v', got: '%v'", c.err, err) + } + }) + } + }) +} + +func ExampleUnmarshal_partial() { + TestBinary := []byte{ + 0x1a, 0x45, 0xdf, 0xa3, 0x84, // EBML + 0x42, 0x87, 0x81, 0x02, // DocTypeVersion = 2 + 0x18, 0x53, 0x80, 0x67, 0xFF, // Segment + 0x16, 0x54, 0xae, 0x6b, 0x85, // Tracks + 0xae, 0x83, // TrackEntry[0] + 0xd7, 0x81, 0x01, // TrackNumber=1 + 0x1F, 0x43, 0xB6, 0x75, 0xFF, // Cluster + 0xE7, 0x81, 0x00, // Timecode + 0xA3, 0x86, 0x81, 0x00, 0x00, 0x88, 0xAA, 0xCC, // SimpleBlock + } + + type TestHeader struct { + Header map[string]interface{} `ebml:"EBML"` + Segment struct { + Tracks map[string]interface{} `ebml:"Tracks,stop"` // Stop unmarshalling after reading this element + } + } + type TestClusters struct { + Cluster []struct { + Timecode uint64 + SimpleBlock []Block + } + } + + r := bytes.NewReader(TestBinary) + + var header TestHeader + if err := Unmarshal(r, &header); !errs.Is(err, ErrReadStopped) { + panic("Unmarshal failed") + } + fmt.Printf("First unmarshal: %v\n", header) + + var clusters TestClusters + if err := Unmarshal(r, &clusters); err != nil { + panic("Unmarshal failed") + } + fmt.Printf("Second unmarshal: %v\n", clusters) + + // Output: + // First unmarshal: {map[EBMLDocTypeVersion:2] {map[TrackEntry:map[TrackNumber:1]]}} + // Second unmarshal: {[{0 [{1 0 true true 0 false [[170 204]]}]}]} +} + +func BenchmarkUnmarshal(b *testing.B) { + TestBinary := []byte{ + 0x1a, 0x45, 0xdf, 0xa3, // EBML + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, // 0x10 + 0x42, 0x82, 0x85, 0x77, 0x65, 0x62, 0x6d, 0x00, // DocType = webm + 0x42, 0x87, 0x81, 0x02, // DocTypeVersion = 2 + 0x42, 0x85, 0x81, 0x02, // DocTypeReadVersion = 2 + 0x18, 0x53, 0x80, 0x67, 0xFF, // Segment + 0x1F, 0x43, 0xB6, 0x75, 0xFF, // Cluster + 0xE7, 0x81, 0x00, // Timecode + 0xA3, 0x86, 0x81, 0x00, 0x00, 0x88, 0xAA, 0xCC, // SimpleBlock + 0xA3, 0x86, 0x81, 0x00, 0x10, 0x88, 0xAA, 0xCC, // SimpleBlock + 0xA3, 0x86, 0x81, 0x00, 0x20, 0x88, 0xAA, 0xCC, // SimpleBlock + 0x1F, 0x43, 0xB6, 0x75, 0xFF, // Cluster + 0xE7, 0x81, 0x10, // Timecode + 0xA3, 0x86, 0x81, 0x00, 0x00, 0x88, 0xAA, 0xCC, // SimpleBlock + 0xA3, 0x86, 0x81, 0x00, 0x10, 0x88, 0xAA, 0xCC, // SimpleBlock + 0xA3, 0x86, 0x81, 0x00, 0x20, 0x88, 0xAA, 0xCC, // SimpleBlock + } + type TestEBML struct { + Header struct { + DocType string `ebml:"EBMLDocType"` + DocTypeVersion uint64 `ebml:"EBMLDocTypeVersion"` + DocTypeReadVersion uint64 `ebml:"EBMLDocTypeReadVersion"` + } `ebml:"EBML"` + Segment []struct { + Cluster struct { + Timecode uint64 + SimpleBlock []Block + } + } + } + + var ret TestEBML + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := Unmarshal(bytes.NewReader(TestBinary), &ret); err != nil { + b.Fatalf("Unexpected error: '%v'", err) + } + } +} diff --git a/pkg/ebml-go/value.go b/pkg/ebml-go/value.go new file mode 100644 index 0000000..81e63c2 --- /dev/null +++ b/pkg/ebml-go/value.go @@ -0,0 +1,482 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ebml + +import ( + "bytes" + "encoding/binary" + "errors" + "io" + "math" + "strings" + "time" +) + +const ( + // DateEpochInUnixtime is the Unixtime of EBML date epoch. + DateEpochInUnixtime = 978307200 + // SizeUnknown is the longest unknown size value. + SizeUnknown = 0xffffffffffffff +) + +// ErrInvalidFloatSize means that a element size is invalid for float type. Float must be 4 or 8 bytes. +var ErrInvalidFloatSize = errors.New("invalid float size") + +// ErrInvalidType means that a value is not convertible to the element data. +var ErrInvalidType = errors.New("invalid type") + +// ErrUnsupportedElementID means that a value is out of range of EBML encoding. +var ErrUnsupportedElementID = errors.New("unsupported Element ID") + +// ErrOutOfRange means that a value is out of range of the data type. +var ErrOutOfRange = errors.New("out of range") + +// valueDecoder is a value decoder sharing internal buffer. +// Member functions must not called concurrently. +type valueDecoder struct { + bs [1]byte +} + +func (d *valueDecoder) decode(t DataType, r io.Reader, n uint64) (interface{}, error) { + switch t { + case DataTypeInt: + return d.readInt(r, n) + case DataTypeUInt: + return d.readUInt(r, n) + case DataTypeDate: + return d.readDate(r, n) + case DataTypeFloat: + return d.readFloat(r, n) + case DataTypeBinary: + return d.readBinary(r, n) + case DataTypeString: + return d.readString(r, n) + case DataTypeBlock: + return d.readBlock(r, n) + } + panic("invalid data type") +} + +func (d *valueDecoder) readDataSize(r io.Reader) (uint64, int, error) { + v, n, err := d.readVUInt(r) + if v == (uint64(0xFFFFFFFFFFFFFFFF) >> uint(64-n*7)) { + return SizeUnknown, n, err + } + return v, n, err +} + +func (d *valueDecoder) readVUInt(r io.Reader) (uint64, int, error) { + bytesRead, err := r.Read(d.bs[:]) + switch err { + case nil: + case io.EOF: + return 0, bytesRead, io.ErrUnexpectedEOF + default: + return 0, bytesRead, err + } + + var vc int + var value uint64 + + b := d.bs[0] + switch { + case b&0x80 == 0x80: + vc = 0 + value = uint64(b & 0x7F) + case b&0xC0 == 0x40: + vc = 1 + value = uint64(b & 0x3F) + case b&0xE0 == 0x20: + vc = 2 + value = uint64(b & 0x1F) + case b&0xF0 == 0x10: + vc = 3 + value = uint64(b & 0x0F) + case b&0xF8 == 0x08: + vc = 4 + value = uint64(b & 0x07) + case b&0xFC == 0x04: + vc = 5 + value = uint64(b & 0x03) + case b&0xFE == 0x02: + vc = 6 + value = uint64(b & 0x01) + case b == 0x01: + vc = 7 + value = 0 + } + + for { + if vc == 0 { + return value, bytesRead, nil + } + + n, err := r.Read(d.bs[:]) + switch err { + case nil: + case io.EOF: + return 0, bytesRead, io.ErrUnexpectedEOF + default: + return 0, bytesRead, err + } + bytesRead += n + value = value<<8 | uint64(d.bs[0]) + vc-- + } +} + +func (d *valueDecoder) readVInt(r io.Reader) (int64, int, error) { + u, n, err := d.readVUInt(r) + if err != nil { + return 0, n, err + } + v := int64(u) + switch n { + case 1: + v -= 0x3F + case 2: + v -= 0x1FFF + case 3: + v -= 0x0FFFFF + case 4: + v -= 0x07FFFFFF + case 5: + v -= 0x03FFFFFFFF + case 6: + v -= 0x01FFFFFFFFFF + case 7: + v -= 0x00FFFFFFFFFFFF + default: + v -= 0x007FFFFFFFFFFFFF + } + return v, n, nil +} + +func (d *valueDecoder) readBinary(r io.Reader, n uint64) (interface{}, error) { + bs := make([]byte, n) + + switch _, err := io.ReadFull(r, bs); err { + case nil: + return bs, nil + case io.EOF: + return bs, io.ErrUnexpectedEOF + default: + return []byte{}, err + } +} + +func (d *valueDecoder) readString(r io.Reader, n uint64) (interface{}, error) { + bs, err := d.readBinary(r, n) + if err != nil { + return "", err + } + s := string(bs.([]byte)) + // Remove trailing null characters + ss := strings.Split(s, "\x00") + return ss[0], nil +} + +func (d *valueDecoder) readInt(r io.Reader, n uint64) (interface{}, error) { + v, err := d.readUInt(r, n) + if err != nil { + return 0, err + } + v64 := v.(uint64) + if n != 8 && (v64&(1<<(n*8-1))) != 0 { + // negative value + for i := n; i < 8; i++ { + v64 |= 0xFF << (i * 8) + } + } + return int64(v64), nil +} + +func (d *valueDecoder) readUInt(r io.Reader, n uint64) (interface{}, error) { + bs := make([]byte, n) + + switch _, err := io.ReadFull(r, bs); err { + case nil: + case io.EOF: + return 0, io.ErrUnexpectedEOF + default: + return 0, err + } + + var v uint64 + for _, b := range bs { + v = v<<8 | uint64(b) + } + return v, nil +} + +func (d *valueDecoder) readDate(r io.Reader, n uint64) (interface{}, error) { + i, err := d.readInt(r, n) + if err != nil { + return time.Unix(0, 0), err + } + return time.Unix(DateEpochInUnixtime, i.(int64)), nil +} + +func (d *valueDecoder) readFloat(r io.Reader, n uint64) (interface{}, error) { + bs := make([]byte, n) + + switch _, err := io.ReadFull(r, bs); err { + case nil: + case io.EOF: + return bs, io.ErrUnexpectedEOF + default: + return []byte{}, err + } + + switch n { + case 4: + return float64(math.Float32frombits(binary.BigEndian.Uint32(bs))), nil + case 8: + return math.Float64frombits(binary.BigEndian.Uint64(bs)), nil + default: + return 0.0, wrapErrorf(ErrInvalidFloatSize, "reading %d bytes float", n) + } +} + +func (d *valueDecoder) readBlock(r io.Reader, n uint64) (interface{}, error) { + b, err := UnmarshalBlock(r, int64(n)) + if err != nil { + return nil, err + } + return *b, nil +} + +var perTypeEncoder = map[DataType]func(interface{}, uint64) ([]byte, error){ + DataTypeInt: encodeInt, + DataTypeUInt: encodeUInt, + DataTypeDate: encodeDate, + DataTypeFloat: encodeFloat, + DataTypeBinary: encodeBinary, + DataTypeString: encodeString, + DataTypeBlock: encodeBlock, +} + +func encodeDataSize(v, n uint64) []byte { + switch { + case v < 0x80-1 && n < 2: + return []byte{byte(v) | 0x80} + case v < 0x4000-1 && n < 3: + return []byte{byte(v>>8) | 0x40, byte(v)} + case v < 0x200000-1 && n < 4: + return []byte{byte(v>>16) | 0x20, byte(v >> 8), byte(v)} + case v < 0x10000000-1 && n < 5: + return []byte{byte(v>>24) | 0x10, byte(v >> 16), byte(v >> 8), byte(v)} + case v < 0x800000000-1 && n < 6: + return []byte{byte(v>>32) | 0x8, byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)} + case v < 0x40000000000-1 && n < 7: + return []byte{byte(v>>40) | 0x4, byte(v >> 32), byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)} + case v < 0x2000000000000-1 && n < 8: + return []byte{byte(v>>48) | 0x2, byte(v >> 40), byte(v >> 32), byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)} + case v < SizeUnknown: + return []byte{0x1, byte(v >> 48), byte(v >> 40), byte(v >> 32), byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)} + default: + return []byte{0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} + } +} + +func encodeElementID(v uint64) ([]byte, error) { + switch { + case v < 0x80: + return []byte{byte(v) | 0x80}, nil + case v < 0x4000: + return []byte{byte(v>>8) | 0x40, byte(v)}, nil + case v < 0x200000: + return []byte{byte(v>>16) | 0x20, byte(v >> 8), byte(v)}, nil + case v < 0x10000000: + return []byte{byte(v>>24) | 0x10, byte(v >> 16), byte(v >> 8), byte(v)}, nil + case v < 0x800000000: + return []byte{byte(v>>32) | 0x8, byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)}, nil + case v < 0x40000000000: + return []byte{byte(v>>40) | 0x4, byte(v >> 32), byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)}, nil + case v < 0x2000000000000: + return []byte{byte(v>>48) | 0x2, byte(v >> 40), byte(v >> 32), byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)}, nil + } + return nil, ErrUnsupportedElementID +} + +func encodeVInt(v int64) ([]byte, error) { + switch { + case -0x3F <= v && v <= 0x3F: + v += 0x3F + return encodeDataSize(uint64(v), 1), nil + case -0x1FFF <= v && v <= 0x1FFF: + v += 0x1FFF + return encodeDataSize(uint64(v), 2), nil + case -0xFFFFF <= v && v <= 0xFFFFF: + v += 0xFFFFF + return encodeDataSize(uint64(v), 3), nil + case -0x7FFFFFF <= v && v <= 0x7FFFFFF: + v += 0x7FFFFFF + return encodeDataSize(uint64(v), 4), nil + case -0x3FFFFFFFF <= v && v <= 0x3FFFFFFFF: + v += 0x3FFFFFFFF + return encodeDataSize(uint64(v), 5), nil + case -0x1FFFFFFFFFF <= v && v <= 0x1FFFFFFFFFF: + v += 0x1FFFFFFFFFF + return encodeDataSize(uint64(v), 6), nil + case -0xFFFFFFFFFFFF <= v && v <= 0xFFFFFFFFFFFF: + v += 0xFFFFFFFFFFFF + return encodeDataSize(uint64(v), 7), nil + case -0x7FFFFFFFFFFFFF <= v && v <= 0x7FFFFFFFFFFFFF: + v += 0x7FFFFFFFFFFFFF + return encodeDataSize(uint64(v), 8), nil + default: + return nil, ErrOutOfRange + } +} + +func encodeBinary(i interface{}, n uint64) ([]byte, error) { + v, ok := i.([]byte) + if !ok { + return []byte{}, ErrInvalidType + } + if uint64(len(v)) >= n { + return v, nil + } + return append(v, bytes.Repeat([]byte{0x00}, int(n)-len(v))...), nil +} + +func encodeString(i interface{}, n uint64) ([]byte, error) { + v, ok := i.(string) + if !ok { + return []byte{}, wrapErrorf(ErrInvalidType, "writing %T as string", i) + } + if uint64(len(v)) >= n { + return append([]byte(v)), nil + } + return append([]byte(v), bytes.Repeat([]byte{0x00}, int(n)-len(v))...), nil +} + +func encodeInt(i interface{}, n uint64) ([]byte, error) { + var v int64 + switch v2 := i.(type) { + case int: + v = int64(v2) + case int8: + v = int64(v2) + case int16: + v = int64(v2) + case int32: + v = int64(v2) + case int64: + v = v2 + default: + return []byte{}, wrapErrorf(ErrInvalidType, "writing %T as int", i) + } + return encodeUInt(uint64(v), n) +} + +func encodeUInt(i interface{}, n uint64) ([]byte, error) { + var v uint64 + switch v2 := i.(type) { + case uint: + v = uint64(v2) + case uint8: + v = uint64(v2) + case uint16: + v = uint64(v2) + case uint32: + v = uint64(v2) + case uint64: + v = v2 + default: + return []byte{}, wrapErrorf(ErrInvalidType, "writing %T as uint", i) + } + switch { + case v < 0x100 && n < 2: + return []byte{byte(v)}, nil + case v < 0x10000 && n < 3: + return []byte{byte(v >> 8), byte(v)}, nil + case v < 0x1000000 && n < 4: + return []byte{byte(v >> 16), byte(v >> 8), byte(v)}, nil + case v < 0x100000000 && n < 5: + return []byte{byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)}, nil + case v < 0x10000000000 && n < 6: + return []byte{byte(v >> 32), byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)}, nil + case v < 0x1000000000000 && n < 7: + return []byte{byte(v >> 40), byte(v >> 32), byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)}, nil + case v < 0x100000000000000 && n < 8: + return []byte{byte(v >> 48), byte(v >> 40), byte(v >> 32), byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)}, nil + default: + return []byte{byte(v >> 56), byte(v >> 48), byte(v >> 40), byte(v >> 32), byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)}, nil + } +} + +func encodeDate(i interface{}, n uint64) ([]byte, error) { + v, ok := i.(time.Time) + if !ok { + return []byte{}, wrapErrorf(ErrInvalidType, "writing %T as date", i) + } + dtns := v.Sub(time.Unix(DateEpochInUnixtime, 0)).Nanoseconds() + return encodeInt(int64(dtns), n) +} + +func encodeFloat32(i float32) ([]byte, error) { + b := make([]byte, 4) + binary.BigEndian.PutUint32(b, math.Float32bits(i)) + return b, nil +} + +func encodeFloat64(i float64) ([]byte, error) { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, math.Float64bits(i)) + return b, nil +} + +func encodeFloat(i interface{}, n uint64) ([]byte, error) { + switch v := i.(type) { + case float64: + switch n { + case 0: + return encodeFloat64(v) + case 4: + return encodeFloat32(float32(v)) + case 8: + return encodeFloat64(v) + default: + return []byte{}, wrapErrorf(ErrInvalidFloatSize, "writing %d bytes float", n) + } + case float32: + switch n { + case 0: + return encodeFloat32(v) + case 4: + return encodeFloat32(v) + case 8: + return encodeFloat64(float64(v)) + default: + return []byte{}, wrapErrorf(ErrInvalidFloatSize, "writing %d bytes float", n) + } + default: + return []byte{}, wrapErrorf(ErrInvalidType, "writing %T as float", i) + } +} + +func encodeBlock(i interface{}, n uint64) ([]byte, error) { + v, ok := i.(Block) + if !ok { + return []byte{}, wrapErrorf(ErrInvalidType, "writing %T as block", i) + } + var b bytes.Buffer + if err := MarshalBlock(&v, &b); err != nil { + return []byte{}, err + } + return b.Bytes(), nil +} diff --git a/pkg/ebml-go/value_test.go b/pkg/ebml-go/value_test.go new file mode 100644 index 0000000..89b8a9b --- /dev/null +++ b/pkg/ebml-go/value_test.go @@ -0,0 +1,429 @@ +package ebml + +import ( + "bytes" + "io" + "reflect" + "testing" + "time" + + "github.com/at-wat/ebml-go/internal/errs" +) + +func TestDataSize(t *testing.T) { + testCases := map[string]struct { + b []byte + i uint64 + }{ + "1 byte (upper bound)": {[]byte{0xFE}, 0x80 - 2}, + "2 bytes (lower bound)": {[]byte{0x40, 0x7F}, 0x80 - 1}, + "2 bytes (upper bound)": {[]byte{0x7F, 0xFE}, 0x4000 - 2}, + "3 bytes (lower bound)": {[]byte{0x20, 0x3F, 0xFF}, 0x4000 - 1}, + "3 bytes (upper bound)": {[]byte{0x3F, 0xFF, 0xFE}, 0x200000 - 2}, + "4 bytes (lower bound)": {[]byte{0x10, 0x1F, 0xFF, 0xFF}, 0x200000 - 1}, + "4 bytes (upper bound)": {[]byte{0x1F, 0xFF, 0xFF, 0xFE}, 0x10000000 - 2}, + "5 bytes (lower bound)": {[]byte{0x08, 0x0F, 0xFF, 0xFF, 0xFF}, 0x10000000 - 1}, + "5 bytes (upper bound)": {[]byte{0x0F, 0xFF, 0xFF, 0xFF, 0xFE}, 0x800000000 - 2}, + "6 bytes (lower bound)": {[]byte{0x04, 0x07, 0xFF, 0xFF, 0xFF, 0xFF}, 0x800000000 - 1}, + "6 bytes (upper bound)": {[]byte{0x07, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE}, 0x40000000000 - 2}, + "7 bytes (lower bound)": {[]byte{0x02, 0x03, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, 0x40000000000 - 1}, + "7 bytes (upper bound)": {[]byte{0x03, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE}, 0x2000000000000 - 2}, + "8 bytes (lower bound)": {[]byte{0x01, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, 0x2000000000000 - 1}, + "8 bytes (upper bound)": {[]byte{0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE}, 0xffffffffffffff - 1}, + "Indefinite": {[]byte{0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, SizeUnknown}, + } + + vd := &valueDecoder{} + + for n, c := range testCases { + t.Run("DecodeVInt "+n, func(t *testing.T) { + r, _, err := vd.readVUInt(bytes.NewBuffer(c.b)) + if err != nil { + t.Fatalf("Failed to readVUInt: '%v'", err) + } + if r != c.i { + t.Errorf("Expected readVUInt result: %d, got: %d", c.i, r) + } + }) + } + for n, c := range testCases { + t.Run("DecodeDataSize "+n, func(t *testing.T) { + r, _, err := vd.readDataSize(bytes.NewBuffer(c.b)) + if err != nil { + t.Fatalf("Failed to readDataSize: '%v'", err) + } + if r != c.i { + t.Errorf("Expected readVUInt result: %d, got: %d", c.i, r) + } + }) + } + for n, c := range testCases { + t.Run("Encode "+n, func(t *testing.T) { + b := encodeDataSize(c.i, 0) + if !bytes.Equal(b, c.b) { + t.Errorf("Expected encodeDataSize result: %d, got: %d", c.b, b) + } + }) + } +} + +func TestDataSize_Unknown(t *testing.T) { + testCases := map[string][]byte{ + "1 byte": {0xFF}, + "2 bytes": {0x7F, 0xFF}, + "3 bytes": {0x3F, 0xFF, 0xFF}, + "4 bytes": {0x1F, 0xFF, 0xFF, 0xFF}, + "5 bytes": {0x0F, 0xFF, 0xFF, 0xFF, 0xFF}, + "6 bytes": {0x07, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + "7 bytes": {0x03, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + "8 bytes": {0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + } + + vd := &valueDecoder{} + + for n, b := range testCases { + t.Run("DecodeDataSize "+n, func(t *testing.T) { + r, _, err := vd.readDataSize(bytes.NewBuffer(b)) + if err != nil { + t.Fatalf("Failed to readDataSize: '%v'", err) + } + if r != SizeUnknown { + t.Errorf("Expected readDataSize result: %d, got: %d", SizeUnknown, r) + } + }) + } +} + +func TestElementID(t *testing.T) { + testCases := map[string]struct { + b []byte + i uint64 + }{ + "1 byte (upper bound)": {[]byte{0xFF}, 0x80 - 1}, + "2 bytes (lower bound)": {[]byte{0x40, 0x80}, 0x80}, + "2 bytes (upper bound)": {[]byte{0x7F, 0xFF}, 0x4000 - 1}, + "3 bytes (lower bound)": {[]byte{0x20, 0x40, 0x00}, 0x4000}, + "3 bytes (upper bound)": {[]byte{0x3F, 0xFF, 0xFF}, 0x200000 - 1}, + "4 bytes (lower bound)": {[]byte{0x10, 0x20, 0x00, 0x00}, 0x200000}, + "4 bytes (upper bound)": {[]byte{0x1F, 0xFF, 0xFF, 0xFF}, 0x10000000 - 1}, + "5 bytes (lower bound)": {[]byte{0x08, 0x10, 0x00, 0x00, 0x00}, 0x10000000}, + "5 bytes (upper bound)": {[]byte{0x0F, 0xFF, 0xFF, 0xFF, 0xFF}, 0x800000000 - 1}, + "6 bytes (lower bound)": {[]byte{0x04, 0x08, 0x00, 0x00, 0x00, 0x00}, 0x800000000}, + "6 bytes (upper bound)": {[]byte{0x07, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, 0x40000000000 - 1}, + "7 bytes (lower bound)": {[]byte{0x02, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00}, 0x40000000000}, + "7 bytes (upper bound)": {[]byte{0x03, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, 0x2000000000000 - 1}, + } + + vd := &valueDecoder{} + + for n, c := range testCases { + t.Run("Decode "+n, func(t *testing.T) { + r, _, err := vd.readVUInt(bytes.NewBuffer(c.b)) + if err != nil { + t.Fatalf("Failed to readVUInt: '%v'", err) + } + if r != c.i { + t.Errorf("Expected readVUInt result: %d, got: %d", c.i, r) + } + }) + } + for n, c := range testCases { + t.Run("Encode "+n, func(t *testing.T) { + b, err := encodeElementID(c.i) + if err != nil { + t.Fatalf("Failed to encodeElementID: '%v'", err) + } + if !bytes.Equal(b, c.b) { + t.Errorf("Expected encodeDataSize result: %d, got: %d", c.b, b) + } + }) + } + + _, err := encodeElementID(0x2000000000000) + if err != ErrUnsupportedElementID { + t.Errorf("Expected error type result: %s, got: %s", ErrUnsupportedElementID, err) + } +} + +func TestVInt(t *testing.T) { + testCases := map[string]struct { + b []byte + i int64 + }{ + "1 byte (lower bound)": {[]byte{0x80}, -0x3F}, + "1 byte (upper bound)": {[]byte{0xFE}, 0x3F}, + "2 bytes (lower bound)": {[]byte{0x40, 0x00}, -0x1FFF}, + "2 bytes (upper bound)": {[]byte{0x7F, 0xFE}, 0x1FFF}, + "3 bytes (lower bound)": {[]byte{0x20, 0x00, 0x00}, -0xFFFFF}, + "3 bytes (upper bound)": {[]byte{0x3F, 0xFF, 0xFE}, 0xFFFFF}, + "4 bytes (lower bound)": {[]byte{0x10, 0x00, 0x00, 0x00}, -0x7FFFFFF}, + "4 bytes (upper bound)": {[]byte{0x1F, 0xFF, 0xFF, 0xFE}, 0x7FFFFFF}, + "5 bytes (lower bound)": {[]byte{0x08, 0x00, 0x00, 0x00, 0x00}, -0x3FFFFFFFF}, + "5 bytes (upper bound)": {[]byte{0x0F, 0xFF, 0xFF, 0xFF, 0xFE}, 0x3FFFFFFFF}, + "6 bytes (lower bound)": {[]byte{0x04, 0x00, 0x00, 0x00, 0x00, 0x00}, -0x1FFFFFFFFFF}, + "6 bytes (upper bound)": {[]byte{0x07, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE}, 0x1FFFFFFFFFF}, + "7 bytes (lower bound)": {[]byte{0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, -0xFFFFFFFFFFFF}, + "7 bytes (upper bound)": {[]byte{0x03, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE}, 0xFFFFFFFFFFFF}, + "8 bytes (lower bound)": {[]byte{0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, -0x7FFFFFFFFFFFFF}, + "8 bytes (upper bound)": {[]byte{0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE}, 0x7FFFFFFFFFFFFF}, + } + + vd := &valueDecoder{} + + for n, c := range testCases { + t.Run("Decode "+n, func(t *testing.T) { + r, _, err := vd.readVInt(bytes.NewBuffer(c.b)) + if err != nil { + t.Fatalf("Failed to readVUInt: '%v'", err) + } + if r != c.i { + t.Errorf("Expected readVUInt result: %d, got: %d", c.i, r) + } + }) + } + for n, c := range testCases { + t.Run("Encode "+n, func(t *testing.T) { + b, err := encodeVInt(c.i) + if err != nil { + t.Fatalf("Failed to encodeVInt: '%v'", err) + } + if !bytes.Equal(b, c.b) { + t.Errorf("Expected encodeVInt result: %d, got: %d", c.b, b) + } + }) + } +} + +func TestValue(t *testing.T) { + testCases := map[string]struct { + b []byte + t DataType + v interface{} + n uint64 + vEnc interface{} + }{ + "Binary": {[]byte{0x01, 0x02, 0x03}, DataTypeBinary, []byte{0x01, 0x02, 0x03}, 0, nil}, + "Binary(4B)": {[]byte{0x01, 0x02, 0x03, 0x00}, DataTypeBinary, []byte{0x01, 0x02, 0x03, 0x00}, 4, []byte{0x01, 0x02, 0x03}}, + "String": {[]byte{0x31, 0x32}, DataTypeString, "12", 0, nil}, + "String(3B)": {[]byte{0x31, 0x32, 0x00}, DataTypeString, "12", 3, nil}, + "String(4B)": {[]byte{0x31, 0x32, 0x00, 0x00}, DataTypeString, "12", 4, nil}, + "Int8": {[]byte{0x01}, DataTypeInt, int64(0x01), 0, nil}, + "Int16": {[]byte{0x01, 0x02}, DataTypeInt, int64(0x0102), 0, nil}, + "Int24": {[]byte{0x01, 0x02, 0x03}, DataTypeInt, int64(0x010203), 0, nil}, + "Int32": {[]byte{0x01, 0x02, 0x03, 0x04}, DataTypeInt, int64(0x01020304), 0, nil}, + "Int40": {[]byte{0x01, 0x02, 0x03, 0x04, 0x05}, DataTypeInt, int64(0x0102030405), 0, nil}, + "Int48": {[]byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, DataTypeInt, int64(0x010203040506), 0, nil}, + "Int56": {[]byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07}, DataTypeInt, int64(0x01020304050607), 0, nil}, + "Int64": {[]byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}, DataTypeInt, int64(0x0102030405060708), 0, nil}, + "Int8(1B)": {[]byte{0x01}, DataTypeInt, int64(0x01), 1, nil}, + "Int16(2B)": {[]byte{0x01, 0x02}, DataTypeInt, int64(0x0102), 2, nil}, + "Int24(3B)": {[]byte{0x01, 0x02, 0x03}, DataTypeInt, int64(0x010203), 3, nil}, + "Int32(4B)": {[]byte{0x01, 0x02, 0x03, 0x04}, DataTypeInt, int64(0x01020304), 4, nil}, + "Int40(5B)": {[]byte{0x01, 0x02, 0x03, 0x04, 0x05}, DataTypeInt, int64(0x0102030405), 5, nil}, + "Int48(6B)": {[]byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, DataTypeInt, int64(0x010203040506), 6, nil}, + "Int56(7B)": {[]byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07}, DataTypeInt, int64(0x01020304050607), 7, nil}, + "Int64(8B)": {[]byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}, DataTypeInt, int64(0x0102030405060708), 8, nil}, + "Int8(2B)": {[]byte{0x00, 0x01}, DataTypeInt, int64(0x01), 2, nil}, + "Int16(3B)": {[]byte{0x00, 0x01, 0x02}, DataTypeInt, int64(0x0102), 3, nil}, + "Int24(4B)": {[]byte{0x00, 0x01, 0x02, 0x03}, DataTypeInt, int64(0x010203), 4, nil}, + "Int32(5B)": {[]byte{0x00, 0x01, 0x02, 0x03, 0x04}, DataTypeInt, int64(0x01020304), 5, nil}, + "Int40(6B)": {[]byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05}, DataTypeInt, int64(0x0102030405), 6, nil}, + "Int48(7B)": {[]byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, DataTypeInt, int64(0x010203040506), 7, nil}, + "Int56(8B)": {[]byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 07}, DataTypeInt, int64(0x01020304050607), 8, nil}, + "UInt": {[]byte{0x01, 0x02, 0x03}, DataTypeUInt, uint64(0x010203), 0, nil}, + "Date": {[]byte{0x01, 0x02, 0x03}, DataTypeDate, time.Unix(DateEpochInUnixtime, 0x010203), 0, nil}, + "Float32": {[]byte{0x40, 0x10, 0x00, 0x00}, DataTypeFloat, float64(2.25), 0, float32(2.25)}, + "Float64": {[]byte{0x40, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, DataTypeFloat, float64(2.25), 0, nil}, + "Float32(8B)": {[]byte{0x40, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, DataTypeFloat, float64(2.25), 8, float32(2.25)}, + "Float64(4B)": {[]byte{0x40, 0x10, 0x00, 0x00}, DataTypeFloat, float64(2.25), 4, float64(2.25)}, + "Float32(4B)": {[]byte{0x40, 0x10, 0x00, 0x00}, DataTypeFloat, float64(2.25), 4, float32(2.25)}, + "Float64(8B)": {[]byte{0x40, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, DataTypeFloat, float64(2.25), 8, nil}, + "Block": {[]byte{0x85, 0x12, 0x34, 0x80, 0x34, 0x56}, DataTypeBlock, + Block{uint64(5), int16(0x1234), true, false, LacingNo, false, [][]byte{{0x34, 0x56}}}, 0, nil, + }, + "ConvertInt8": {[]byte{0x01}, DataTypeInt, int64(0x01), 0, int8(0x01)}, + "ConvertInt16": {[]byte{0x01, 0x02}, DataTypeInt, int64(0x0102), 0, int16(0x0102)}, + "ConvertInt32": {[]byte{0x01, 0x02, 0x03, 0x04}, DataTypeInt, int64(0x01020304), 0, int32(0x01020304)}, + "ConvertInt": {[]byte{0x01, 0x02, 0x03, 0x04}, DataTypeInt, int64(0x01020304), 0, int(0x01020304)}, + "ConvertUInt8": {[]byte{0x01}, DataTypeUInt, uint64(0x01), 0, uint8(0x01)}, + "ConvertUInt16": {[]byte{0x01, 0x02}, DataTypeUInt, uint64(0x0102), 0, uint16(0x0102)}, + "ConvertUInt32": {[]byte{0x01, 0x02, 0x03, 0x04}, DataTypeUInt, uint64(0x01020304), 0, uint32(0x01020304)}, + "ConvertUInt": {[]byte{0x01, 0x02, 0x03, 0x04}, DataTypeUInt, uint64(0x01020304), 0, uint(0x01020304)}, + } + + vd := &valueDecoder{} + + for n, c := range testCases { + t.Run("Read "+n, func(t *testing.T) { + v, err := vd.decode(c.t, bytes.NewBuffer(c.b), uint64(len(c.b))) + if err != nil { + t.Fatalf("Failed to read%s: '%v'", n, err) + } + if !reflect.DeepEqual(v, c.v) { + t.Errorf("Expected read%s result: %v, got: %v", n, c.v, v) + } + }) + t.Run("Encode "+n, func(t *testing.T) { + var v interface{} + if c.vEnc != nil { + v = c.vEnc + } else { + v = c.v + } + b, err := perTypeEncoder[c.t](v, c.n) + if err != nil { + t.Fatalf("Failed to encode%s: '%v'", n, err) + } + if !bytes.Equal(b, c.b) { + t.Errorf("Expected encode%s result: %v, got: %v", n, c.b, b) + } + }) + } +} + +func TestEncodeValue_WrongInputType(t *testing.T) { + testCases := []struct { + t DataType + v []interface{} + err error + }{ + { + DataTypeBinary, + []interface{}{"aaa", int64(1), uint64(1), time.Unix(1, 0), float32(1.0), float64(1.0), Block{}}, + ErrInvalidType, + }, + { + DataTypeString, + []interface{}{[]byte{0x01}, int64(1), uint64(1), time.Unix(1, 0), float32(1.0), float64(1.0), Block{}}, + ErrInvalidType, + }, + { + DataTypeInt, + []interface{}{"aaa", []byte{0x01}, uint64(1), time.Unix(1, 0), float32(1.0), float64(1.0), Block{}}, + ErrInvalidType, + }, + { + DataTypeUInt, + []interface{}{"aaa", []byte{0x01}, int64(1), time.Unix(1, 0), float32(1.0), float64(1.0), Block{}}, + ErrInvalidType, + }, + { + DataTypeDate, + []interface{}{"aaa", []byte{0x01}, int64(1), uint64(1), float32(1.0), float64(1.0), Block{}}, + ErrInvalidType, + }, + { + DataTypeFloat, + []interface{}{"aaa", []byte{0x01}, int64(1), uint64(1), time.Unix(1, 0), Block{}}, + ErrInvalidType, + }, + { + DataTypeBlock, + []interface{}{"aaa", []byte{0x01}, int64(1), uint64(1), time.Unix(1, 0), float32(1.0), float64(1.0)}, + ErrInvalidType, + }, + } + for _, c := range testCases { + t.Run("Encode "+c.t.String(), func(t *testing.T) { + for _, v := range c.v { + _, err := perTypeEncoder[c.t](v, 0) + if !errs.Is(err, c.err) { + t.Fatalf("Expected error against wrong input type %s: '%v, got: '%v'", c.t.String(), c.err, err) + } + } + }) + } +} + +func TestEncodeValue_WrongSize(t *testing.T) { + testCases := map[string]struct { + t DataType + v interface{} + n uint64 + err error + }{ + "Float32(3B)": { + DataTypeFloat, + float32(1.0), + 3, + ErrInvalidFloatSize, + }, + "Float64(9B)": { + DataTypeFloat, + float64(1.0), + 9, + ErrInvalidFloatSize, + }, + } + for n, c := range testCases { + t.Run("Encode "+n, func(t *testing.T) { + _, err := perTypeEncoder[c.t](c.v, c.n) + if !errs.Is(err, c.err) { + t.Fatalf("Expected error against wrong input type %s: '%v', got: '%v'", n, c.err, err) + } + }) + } +} + +func TestEncodeValue_OutOfRange(t *testing.T) { + _, err := encodeVInt(1<<63 - 1) + if err != ErrOutOfRange { + t.Fatalf("Expected error: '%v', got: '%v'", ErrOutOfRange, err) + } +} + +func TestReadValue_WrongSize(t *testing.T) { + testCases := map[string]struct { + t DataType + b []byte + n uint64 + err error + }{ + "Float32(3B)": { + DataTypeFloat, + []byte{0, 0, 0}, + 3, + ErrInvalidFloatSize, + }, + } + + vd := &valueDecoder{} + + for n, c := range testCases { + t.Run("Read "+n, func(t *testing.T) { + _, err := vd.decode(c.t, bytes.NewReader(c.b), c.n) + if !errs.Is(err, c.err) { + t.Fatalf("Expected error against wrong data size of %s: %v, got: %v", n, c.err, err) + } + }) + } +} + +func TestReadValue_ReadUnexpectedEOF(t *testing.T) { + testCases := []struct { + t DataType + b []byte + }{ + {DataTypeBinary, []byte{0x00, 0x00}}, + {DataTypeString, []byte{0x00, 0x00}}, + {DataTypeInt, []byte{0x00, 0x00}}, + {DataTypeUInt, []byte{0x00, 0x00}}, + {DataTypeDate, []byte{0x00, 0x00}}, + {DataTypeFloat, []byte{0x00, 0x00, 0x00, 0x00}}, + } + + vd := &valueDecoder{} + + for _, c := range testCases { + t.Run("Read "+c.t.String(), func(t *testing.T) { + for l := 0; l < len(c.b)-1; l++ { + r := bytes.NewReader(c.b[:l]) + _, err := vd.decode(c.t, r, uint64(len(c.b))) + if !errs.Is(err, io.ErrUnexpectedEOF) { + t.Errorf("Expected error against short (%d bytes) %s: %v, got: %v", + l, c.t.String(), io.ErrUnexpectedEOF, err) + } + } + }) + } +} diff --git a/pkg/ebml-go/webm/blockwriter.go b/pkg/ebml-go/webm/blockwriter.go new file mode 100644 index 0000000..cb6a75a --- /dev/null +++ b/pkg/ebml-go/webm/blockwriter.go @@ -0,0 +1,70 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package webm + +import ( + "io" + "os" + + "github.com/at-wat/ebml-go/mkvcore" +) + +// NewSimpleBlockWriter creates BlockWriteCloser for each track specified as tracks argument. +// Blocks will be written to WebM as EBML SimpleBlocks. +// Resultant WebM is written to given io.WriteCloser and will be closed automatically; don't close it by yourself. +// Frames written to each track must be sorted by their timestamp. +func NewSimpleBlockWriter(w0 io.WriteCloser, tracks []TrackEntry, opts ...mkvcore.BlockWriterOption) ([]BlockWriteCloser, error) { + trackDesc := []mkvcore.TrackDescription{} + for _, t := range tracks { + trackDesc = append(trackDesc, + mkvcore.TrackDescription{ + TrackNumber: t.TrackNumber, + TrackEntry: t, + }) + } + options := []mkvcore.BlockWriterOption{ + mkvcore.WithEBMLHeader(DefaultEBMLHeader), + mkvcore.WithSegmentInfo(DefaultSegmentInfo), + mkvcore.WithBlockInterceptor(DefaultBlockInterceptor), + } + options = append(options, opts...) + ws, err := mkvcore.NewSimpleBlockWriter(w0, trackDesc, options...) + webmWs := []BlockWriteCloser{} + for _, w := range ws { + webmWs = append(webmWs, BlockWriteCloser(w)) + } + return webmWs, err +} + +// NewSimpleWriter creates BlockWriteCloser for each track specified as tracks argument. +// Blocks will be written to WebM as EBML SimpleBlocks. +// Resultant WebM is written to given io.WriteCloser. +// io.WriteCloser will be closed automatically; don't close it by yourself. +// +// Deprecated: This is exposed to keep compatibility with the old version. +// Use NewSimpleBlockWriter instead. +func NewSimpleWriter(w0 io.WriteCloser, tracks []TrackEntry, opts ...mkvcore.BlockWriterOption) ([]*FrameWriter, error) { + os.Stderr.WriteString( + "Deprecated: You are using deprecated webm.NewSimpleWriter and *webm.blockWriter.\n" + + " Use webm.NewSimpleBlockWriter and webm.BlockWriteCloser interface instead.\n" + + " See https://godoc.org/github.com/at-wat/ebml-go to find out the latest API.\n", + ) + ws, err := NewSimpleBlockWriter(w0, tracks, opts...) + var ws2 []*FrameWriter + for _, w := range ws { + ws2 = append(ws2, &FrameWriter{w}) + } + return ws2, err +} diff --git a/pkg/ebml-go/webm/blockwriter_test.go b/pkg/ebml-go/webm/blockwriter_test.go new file mode 100644 index 0000000..36fe704 --- /dev/null +++ b/pkg/ebml-go/webm/blockwriter_test.go @@ -0,0 +1,194 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package webm + +import ( + "bytes" + "reflect" + "testing" + + "github.com/at-wat/ebml-go" + "github.com/at-wat/ebml-go/internal/buffercloser" + "github.com/at-wat/ebml-go/mkvcore" +) + +func TestBlockWriter(t *testing.T) { + buf := buffercloser.New() + + tracks := []TrackEntry{ + { + Name: "Video", + TrackNumber: 1, + TrackUID: 12345, + CodecID: "V_VP8", + TrackType: 1, + Video: &Video{ + PixelWidth: 320, + PixelHeight: 240, + }, + }, + { + Name: "Audio", + TrackNumber: 2, + TrackUID: 54321, + CodecID: "A_OPUS", + TrackType: 2, + Audio: &Audio{ + SamplingFrequency: 48000.0, + Channels: 2, + }, + }, + } + ws, err := NewSimpleBlockWriter(buf, tracks) + if err != nil { + t.Fatalf("Failed to create BlockWriter: %v", err) + } + + if len(ws) != len(tracks) { + t.Fatalf("Number of the returned writer (%d) must be same as the number of TrackEntry (%d)", len(ws), len(tracks)) + } + + if n, err := ws[0].Write(false, 100, []byte{0x01, 0x02}); err != nil { + t.Fatalf("Failed to Write: %v", err) + } else if n != 2 { + t.Errorf("Expected return value of BlockWriter.Write: 2, got: %d", n) + } + + if n, err := ws[1].Write(true, 110, []byte{0x03, 0x04, 0x05}); err != nil { + t.Fatalf("Failed to Write: %v", err) + } else if n != 3 { + t.Errorf("Expected return value of BlockWriter.Write: 3, got: %d", n) + } + + // Ignored due to old timestamp + if n, err := ws[0].Write(true, -32769, []byte{0x0A}); err != nil { + t.Fatalf("Failed to Write: %v", err) + } else if n != 1 { + t.Errorf("Expected return value of BlockWriter.Write: 1, got: %d", n) + } + + if n, err := ws[0].Write(true, 130, []byte{0x06}); err != nil { + t.Fatalf("Failed to Write: %v", err) + } else if n != 1 { + t.Errorf("Expected return value of BlockWriter.Write: 1, got: %d", n) + } + + ws[0].Close() + ws[1].Close() + select { + case <-buf.Closed(): + default: + t.Errorf("Base io.WriteCloser is not closed by BlockWriter") + } + + expected := struct { + Header EBMLHeader `ebml:"EBML"` + Segment Segment `ebml:"Segment,size=unknown"` + }{ + Header: *DefaultEBMLHeader, + Segment: Segment{ + Info: *DefaultSegmentInfo, + Tracks: Tracks{ + TrackEntry: tracks, + }, + Cluster: []Cluster{ + { + Timecode: uint64(0), + SimpleBlock: []ebml.Block{ + { + TrackNumber: 1, + Timecode: int16(0), + Keyframe: false, + Data: [][]byte{{0x01, 0x02}}, + }, + { + TrackNumber: 2, + Timecode: int16(10), + Keyframe: true, + Data: [][]byte{{0x03, 0x04, 0x05}}, + }, + { + TrackNumber: 1, + Timecode: int16(30), + Keyframe: true, + Data: [][]byte{{0x06}}, + }, + }, + }, + { + Timecode: uint64(30), + PrevSize: uint64(39), + }, + }, + }, + } + var result struct { + Header EBMLHeader `ebml:"EBML"` + Segment Segment `ebml:"Segment,size=unknown"` + } + if err := ebml.Unmarshal(bytes.NewReader(buf.Bytes()), &result); err != nil { + t.Fatalf("Failed to Unmarshal resultant binary: %v", err) + } + if !reflect.DeepEqual(expected, result) { + t.Errorf("Unexpected WebM data,\nexpected: %+v\n got: %+v", expected, result) + } +} + +func TestBlockWriter_NewSimpleWriter(t *testing.T) { + buf := buffercloser.New() + + tracks := []TrackEntry{ + { + TrackNumber: 1, + TrackUID: 2, + CodecID: "", + TrackType: 1, + }, + } + + // Check old API + var ws []*FrameWriter + var err error + + ws, err = NewSimpleWriter( + buf, tracks, + mkvcore.WithEBMLHeader(nil), + mkvcore.WithSegmentInfo(nil), + mkvcore.WithSeekHead(false), + ) + if err != nil { + t.Fatalf("Failed to create BlockWriter: %v", err) + } + + if len(ws) != 1 { + t.Fatalf("Number of the returned writer must be 1, but got %d", len(ws)) + } + ws[0].Close() + + expectedBytes := []byte{ + 0x18, 0x53, 0x80, 0x67, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x16, 0x54, 0xAE, 0x6B, 0x8F, + 0xAE, 0x8D, + 0xD7, 0x81, 0x01, + 0x73, 0xC5, 0x81, 0x02, + 0x86, 0x81, 0x00, + 0x83, 0x81, 0x01, + 0x1F, 0x43, 0xB6, 0x75, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xE7, 0x81, 0x00, + } + if !bytes.Equal(buf.Bytes(), expectedBytes) { + t.Errorf("Unexpected WebM binary,\nexpected: %+v\n got: %+v", expectedBytes, buf.Bytes()) + } +} diff --git a/pkg/ebml-go/webm/const.go b/pkg/ebml-go/webm/const.go new file mode 100644 index 0000000..f999a05 --- /dev/null +++ b/pkg/ebml-go/webm/const.go @@ -0,0 +1,40 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package webm + +import ( + "github.com/at-wat/ebml-go/mkvcore" +) + +var ( + // DefaultEBMLHeader is the default EBML header used by BlockWriter. + DefaultEBMLHeader = &EBMLHeader{ + EBMLVersion: 1, + EBMLReadVersion: 1, + EBMLMaxIDLength: 4, + EBMLMaxSizeLength: 8, + DocType: "webm", + DocTypeVersion: 4, // May contain v4 elements, + DocTypeReadVersion: 2, // and playable by parsing v2 elements. + } + // DefaultSegmentInfo is the default Segment.Info used by BlockWriter. + DefaultSegmentInfo = &Info{ + TimecodeScale: 1000000, // 1ms + MuxingApp: "ebml-go.webm.BlockWriter", + WritingApp: "ebml-go.webm.BlockWriter", + } + // DefaultBlockInterceptor is the default BlockInterceptor used by BlockWriter. + DefaultBlockInterceptor = mkvcore.MustBlockInterceptor(mkvcore.NewMultiTrackBlockSorter(mkvcore.WithMaxDelayedPackets(16), mkvcore.WithSortRule(mkvcore.BlockSorterDropOutdated))) +) diff --git a/pkg/ebml-go/webm/interface.go b/pkg/ebml-go/webm/interface.go new file mode 100644 index 0000000..0a93730 --- /dev/null +++ b/pkg/ebml-go/webm/interface.go @@ -0,0 +1,52 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package webm + +import ( + "github.com/at-wat/ebml-go/mkvcore" +) + +// BlockWriter is a WebM block writer interface. +type BlockWriter interface { + mkvcore.BlockWriter +} + +// BlockReader is a WebM block reader interface. +type BlockReader interface { + mkvcore.BlockReader +} + +// BlockCloser is a WebM closer interface. +type BlockCloser interface { + mkvcore.BlockCloser +} + +// BlockWriteCloser groups Writer and Closer. +type BlockWriteCloser interface { + mkvcore.BlockWriteCloser +} + +// BlockReadCloser groups Reader and Closer. +type BlockReadCloser interface { + mkvcore.BlockReadCloser +} + +// FrameWriter is a backward compatibility wrapper of BlockWriteCloser. +// +// Deprecated: This is exposed to keep compatibility with the old version. +// Use BlockWriteCloser interface instead. +type FrameWriter struct { + mkvcore.BlockWriteCloser +} diff --git a/pkg/ebml-go/webm/webm.go b/pkg/ebml-go/webm/webm.go new file mode 100644 index 0000000..044a560 --- /dev/null +++ b/pkg/ebml-go/webm/webm.go @@ -0,0 +1,137 @@ +// Copyright 2019 The ebml-go authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package webm provides the WebM multimedia writer. +// +// The package implements block data writer for multi-track WebM container. +package webm + +import ( + "time" + + "github.com/at-wat/ebml-go" +) + +// EBMLHeader represents EBML header struct. +type EBMLHeader struct { + EBMLVersion uint64 `ebml:"EBMLVersion"` + EBMLReadVersion uint64 `ebml:"EBMLReadVersion"` + EBMLMaxIDLength uint64 `ebml:"EBMLMaxIDLength"` + EBMLMaxSizeLength uint64 `ebml:"EBMLMaxSizeLength"` + DocType string `ebml:"EBMLDocType"` + DocTypeVersion uint64 `ebml:"EBMLDocTypeVersion"` + DocTypeReadVersion uint64 `ebml:"EBMLDocTypeReadVersion"` +} + +// Seek represents Seek element struct. +type Seek struct { + SeekID []byte `ebml:"SeekID"` + SeekPosition uint64 `ebml:"SeekPosition"` +} + +// SeekHead represents SeekHead element struct. +type SeekHead struct { + Seek []Seek `ebml:"Seek"` +} + +// Info represents Info element struct. +type Info struct { + TimecodeScale uint64 `ebml:"TimecodeScale"` + MuxingApp string `ebml:"MuxingApp,omitempty"` + WritingApp string `ebml:"WritingApp,omitempty"` + Duration float64 `ebml:"Duration,omitempty"` + DateUTC time.Time `ebml:"DateUTC,omitempty"` +} + +// TrackEntry represents TrackEntry element struct. +type TrackEntry struct { + Name string `ebml:"Name,omitempty"` + TrackNumber uint64 `ebml:"TrackNumber"` + TrackUID uint64 `ebml:"TrackUID"` + CodecID string `ebml:"CodecID"` + CodecPrivate []byte `ebml:"CodecPrivate,omitempty"` + CodecDelay uint64 `ebml:"CodecDelay,omitempty"` + TrackType uint64 `ebml:"TrackType"` + DefaultDuration uint64 `ebml:"DefaultDuration,omitempty"` + SeekPreRoll uint64 `ebml:"SeekPreRoll,omitempty"` + Audio *Audio `ebml:"Audio"` + Video *Video `ebml:"Video"` +} + +// Audio represents Audio element struct. +type Audio struct { + SamplingFrequency float64 `ebml:"SamplingFrequency"` + Channels uint64 `ebml:"Channels"` +} + +// Video represents Video element struct. +type Video struct { + PixelWidth uint64 `ebml:"PixelWidth"` + PixelHeight uint64 `ebml:"PixelHeight"` +} + +// Tracks represents Tracks element struct. +type Tracks struct { + TrackEntry []TrackEntry `ebml:"TrackEntry"` +} + +// BlockGroup represents BlockGroup element struct. +type BlockGroup struct { + BlockDuration uint64 `ebml:"BlockDuration,omitempty"` + ReferenceBlock int64 `ebml:"ReferenceBlock,omitempty"` + Block ebml.Block `ebml:"Block"` +} + +// Cluster represents Cluster element struct. +type Cluster struct { + Timecode uint64 `ebml:"Timecode"` + PrevSize uint64 `ebml:"PrevSize,omitempty"` + BlockGroup []BlockGroup `ebml:"BlockGroup"` + SimpleBlock []ebml.Block `ebml:"SimpleBlock"` +} + +// Cues represents Cues element struct. +type Cues struct { + CuePoint []CuePoint `ebml:"CuePoint"` +} + +// CuePoint represents CuePoint element struct. +type CuePoint struct { + CueTime uint64 `ebml:"CueTime"` + CueTrackPositions []CueTrackPosition `ebml:"CueTrackPositions"` +} + +// CueTrackPosition represents CueTrackPosition element struct. +type CueTrackPosition struct { + CueTrack uint64 `ebml:"CueTrack"` + CueClusterPosition uint64 `ebml:"CueClusterPosition"` + CueBlockNumber uint64 `ebml:"CueBlockNumber,omitempty"` +} + +// Segment represents Segment element struct. +type Segment struct { + SeekHead *SeekHead `ebml:"SeekHead"` + Info Info `ebml:"Info"` + Tracks Tracks `ebml:"Tracks"` + Cluster []Cluster `ebml:"Cluster"` + Cues *Cues `ebml:"Cues"` +} + +// SegmentStream represents Segment element struct for streaming. +type SegmentStream struct { + SeekHead *SeekHead `ebml:"SeekHead"` + Info Info `ebml:"Info"` + Tracks Tracks `ebml:"Tracks"` + Cluster []Cluster `ebml:"Cluster,size=unknown"` +} diff --git a/web/public/index.html b/web/public/index.html index 21d04d2..1a31a9b 100644 --- a/web/public/index.html +++ b/web/public/index.html @@ -1,29 +1,39 @@ - - -

- - -[ All Recordings ] -

-Recording mode - -

+ #logs { + font-family: monospace + } + +
+ + +

+ + + [ All Recordings ] +

+ Recording mode + +
+