From 5f8fb6bb3fa53b1f17c719fa548a34b034e28bc4 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Wed, 23 Oct 2024 16:25:55 +1100 Subject: [PATCH 1/9] Check uncompressed and compressedWriter size SxS --- go/go.work | 4 +- go/go.work.sum | 163 ++++++++++++++++++++++++++++++++++++++++- go/mcap/go.mod | 8 +- go/mcap/go.sum | 26 +++++++ go/mcap/reader_test.go | 4 +- go/mcap/writer.go | 53 ++++++++++---- 6 files changed, 234 insertions(+), 24 deletions(-) diff --git a/go/go.work b/go/go.work index 7326675cdf..ce4e5a9e04 100644 --- a/go/go.work +++ b/go/go.work @@ -1,4 +1,6 @@ -go 1.22.5 +go 1.23 + +toolchain go1.23.1 use ( ./cli/mcap diff --git a/go/go.work.sum b/go/go.work.sum index aadcfec65c..064edb2057 100644 --- a/go/go.work.sum +++ b/go/go.work.sum @@ -1,32 +1,187 @@ +cloud.google.com/go/bigquery v1.8.0 h1:PQcPefKFdaIzjQFbiyOgAqyx8q5djaE7x9Sqe712DPA= +cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= +cloud.google.com/go/firestore v1.6.1 h1:8rBq3zRjnHx8UtBvaOWqBB1xq9jH6/wltfQLlTMh2Fw= cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= +cloud.google.com/go/pubsub v1.3.1 h1:ukjixP1wl0LpnZ6LWtZJ0mX5tBmjp1f8Sqer8Z2OMUU= +crawshaw.io/sqlite v0.3.2 h1:N6IzTjkiw9FItHAa0jp+ZKC6tuLzXqAYIv+ccIWos1I= +crawshaw.io/sqlite v0.3.2/go.mod h1:igAO5JulrQ1DbdZdtVq48mnZUBAPOeFzer7VhDWNtW4= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/RoaringBitmap/roaring v0.4.23 h1:gpyfd12QohbqhFO4NVDUdoPOCXsyahYRQhINmlHxKeo= +github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= +github.com/anacrolix/envpprof v1.1.0 h1:hz8QWMN1fA01YNQsUtVvl9hBXQWWMxSnHHoOK9IdrNY= +github.com/anacrolix/envpprof v1.1.0/go.mod h1:My7T5oSqVfEn4MD4Meczkw/f5lSIndGAKu/0SM/rkf4= +github.com/anacrolix/log v0.6.0 h1:5y+wtTWoecbrAWWuoBCH7UuGFiD6q2jnQxrLK01RC+Q= +github.com/anacrolix/log v0.6.0/go.mod h1:lWvLTqzAnCWPJA08T2HCstZi0L1y2Wyvm3FJgwU9jwU= +github.com/anacrolix/missinggo v1.2.1 h1:0IE3TqX5y5D0IxeMwTyIgqdDew4QrzcXaaEnJQyjHvw= +github.com/anacrolix/missinggo v1.2.1/go.mod h1:J5cMhif8jPmFoC3+Uvob3OXXNIhOUikzMt+uUjeM21Y= +github.com/anacrolix/stm v0.2.0 h1:q4IrMj/PPQM2OgHkQcWEVmiOXcuf0Kkil2rWasQeGkU= +github.com/anacrolix/stm v0.2.0/go.mod h1:zoVQRvSiGjGoTmbM0vSLIiaKjWtNPeTvXUSdJQA4hsg= +github.com/anacrolix/tagflag v1.1.0 h1:0lHtP7w9MczBioGf/b4jeQ+OiJWOPfQwPbBPGnkovvU= +github.com/anacrolix/tagflag v1.1.0/go.mod h1:Scxs9CV10NQatSmbyjqmqmeQNwGzlNe0CMUMIxqHIG8= +github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= +github.com/armon/go-metrics v0.3.10 h1:FR+drcQStOe+32sYyJYyZ7FIdgoGGBnwLl+flodp8Uo= github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= +github.com/benbjohnson/immutable v0.2.0 h1:t0rW3lNFwfQ85IDO1mhMbumxdVSti4nnVaal4r45Oio= +github.com/benbjohnson/immutable v0.2.0/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8= +github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og= +github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= +github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 h1:hzAQntlaYRkVSFEfj9OTWlVV1H155FMD8BTKktLv0QI= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1 h1:zH8ljVhhq7yC0MIeUL/IviMtY8hx2mK8cN9wEYb8ggw= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 h1:bWDMxwH3px2JBh6AyO7hdCn/PkvCZXii8TGj7sbtEbQ= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1 h1:xvqufLtNVwAhN8NMyWklVgxnWohi+wtMGQMhtxexlm0= +github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A= github.com/foxglove/mcap/go/mcap v0.0.0-20220328132551-ffb9c0b0ebdc/go.mod h1:gQrB8PzccHW69xedSZ0uVDQVgDd3h1qX+otbS6fjSkE= github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a h1:FQqoVvjbiUioBBFUL5up+h+GdCa/AnJsL/1bIs/veSI= +github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8 h1:tlyzajkF3030q6M8SvmJSemC9DTHL/xaMa18b65+JM4= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/hashicorp/consul/api v1.12.0 h1:k3y1FYv6nuKyNTqj6w9gXOx5r5CfLj/k/euUeBXj1OY= github.com/hashicorp/consul/api v1.12.0/go.mod h1:6pVBMo0ebnYdt2S3H87XhekM/HHrUoTD2XXb/VrZVy0= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.2.0 h1:La19f8d7WIlm4ogzNHB0JGqs5AUDAZ2UfCY4sJXcJdM= github.com/hashicorp/go-hclog v1.2.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/serf v0.9.7 h1:hkdgbqizGQHuU5IPqYM1JdSMV8nKfpuOnZYXssk9muY= github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639 h1:mV02weKRL81bEnm8A0HT1/CAelMQDBuQIfLw8n+d6xI= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= +github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcMpsXCp3lJ03zYT1PkRd3kQGPn9GVg= +github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= +github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= +github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= +github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= +github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/sftp v1.13.1 h1:I2qBYMChEhIjOgazfJmV3/mZM256btk6wkCDRmW7JYs= +github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s= +github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= +github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= +github.com/sagikazarmark/crypt v0.6.0 h1:REOEXCs/NFY/1jOCEouMuT4zEniE5YoXbvpC5X/TLF8= github.com/sagikazarmark/crypt v0.6.0/go.mod h1:U8+INwJo3nBv1m6A/8OBXAq7Jnpspk5AxSgDyEQcea8= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/tinylib/msgp v1.1.2 h1:gWmO7n0Ys2RBEb7GPYB9Ujq8Mk5p2U08lRnmMcGy6BQ= +github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc= +github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs= +go.etcd.io/etcd/api/v3 v3.5.4 h1:OHVyt3TopwtUQ2GKdd5wu3PmmipR4FTwCqoEjSyRdIc= go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= +go.etcd.io/etcd/client/pkg/v3 v3.5.4 h1:lrneYvz923dvC14R54XcA7FXoZ3mlGZAgmwhfm7HqOg= go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.4 h1:Dcx3/MYyfKcPNLpR4VVQUP5KgYrBeJtktBwEKkw08Ao= go.etcd.io/etcd/client/v2 v2.305.4/go.mod h1:Ud+VUwIi9/uQHOMA+4ekToJ12lTxlv0zB/+DHwTGEbU= +go.etcd.io/etcd/client/v3 v3.5.4 h1:p83BUL3tAYS0OT/r0qglgc3M1JjhM0diV8DSWAhVXv4= go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY= +go.opentelemetry.io/proto/otlp v0.7.0 h1:rwOQPCuKAKmwGKq2aVNnYIibI6wnV7EvzgfTCzcdGg8= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs= +golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 h1:LQmS1nU0twXLA96Kt7U9qtHJEbBk3z6Q0V4UXjZkpr4= +golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= +golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= +golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 h1:0c3L82FDQ5rt1bjTBlchS8t6RQ6299/+5bWMnRLh+uI= +golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= +golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 h1:M1YKkFIboKNieVO5DLUEVzQfGwJD30Nv2jfUgzb5UcE= +gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= +honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= +modernc.org/libc v1.21.5 h1:xBkU9fnHV+hvZuPSRszN0AXDG4M7nwPLwTWwkYcvLCI= +modernc.org/libc v1.21.5/go.mod h1:przBsL5RDOZajTVslkugzLBj1evTue36jEomFQOoYuI= +modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.4.0 h1:crykUfNSnMAXaOJnnxcSzbUGMqkLWjklJKkBK2nwZwk= +modernc.org/memory v1.4.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/sqlite v1.20.0 h1:80zmD3BGkm8BZ5fUi/4lwJQHiO3GXgIUvZRXpoIfROY= +modernc.org/sqlite v1.20.0/go.mod h1:EsYz8rfOvLCiYTy5ZFsOYzoCcRMu98YYkwAcCw5YIYw= +rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= +rsc.io/quote/v3 v3.1.0 h1:9JKUTTIUgS6kzR9mK1YuGKv6Nl+DijDNIc0ghT58FaY= +rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4= +zombiezen.com/go/sqlite v0.12.0 h1:0IDiV/XR6fWS2iFcOuVpGg3O2rJV0uVYEW30ANTKjeE= +zombiezen.com/go/sqlite v0.12.0/go.mod h1:RKdRR9xoQDSnB47yy7G4PtrjGZJtupb/SyEbJZLaRes= diff --git a/go/mcap/go.mod b/go/mcap/go.mod index d4ad84fc17..85505ede63 100644 --- a/go/mcap/go.mod +++ b/go/mcap/go.mod @@ -1,17 +1,21 @@ module github.com/foxglove/mcap/go/mcap -go 1.18 +go 1.23 require ( + github.com/anacrolix/missinggo/v2 v2.8.0 github.com/klauspost/compress v1.16.7 github.com/pierrec/lz4/v4 v4.1.21 github.com/stretchr/testify v1.9.0 ) require ( + github.com/anacrolix/generics v0.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/kr/pretty v0.3.0 // indirect + github.com/huandu/xstrings v1.3.2 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 // indirect gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go/mcap/go.sum b/go/mcap/go.sum index 9b52a5a7c1..ee8e9c76bc 100644 --- a/go/mcap/go.sum +++ b/go/mcap/go.sum @@ -1,12 +1,38 @@ +github.com/anacrolix/generics v0.0.2 h1:UbtD+KntUGxeGYMC4RwhsETieL9ixGdSptJQRhdy7No= +github.com/anacrolix/generics v0.0.2/go.mod h1:ff2rHB/joTV03aMSSn/AZNnaIpUw0h3njetGsaXcMy8= +github.com/anacrolix/missinggo/v2 v2.8.0 h1:6pGnVOlR6TWL9JM5Msyezij8YHU3+oHO7r82Eql/kpA= +github.com/anacrolix/missinggo/v2 v2.8.0/go.mod h1:vVO5FEziQm+NFmJesc7StpkquZk+WJFCaL0Wp//2sa0= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= +github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/exp v0.0.0-20220428152302-39d4317da171 h1:TfdoLivD44QwvssI9Sv1xwa5DcL5XQr4au4sZ2F2NV4= +golang.org/x/exp v0.0.0-20220428152302-39d4317da171/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= +golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go/mcap/reader_test.go b/go/mcap/reader_test.go index e1322f16a7..a463ecc2ca 100644 --- a/go/mcap/reader_test.go +++ b/go/mcap/reader_test.go @@ -772,7 +772,7 @@ func TestReadingMessageOrderWithOverlappingChunks(t *testing.T) { } var now uint64 = 100 addMsg(now) - for writer.compressedWriter.Size() != 0 { + for writer.uncompressedSize() != 0 { now += 10 addMsg(now) } @@ -782,7 +782,7 @@ func TestReadingMessageOrderWithOverlappingChunks(t *testing.T) { now -= 55 addMsg(now) - for writer.compressedWriter.Size() != 0 { + for writer.uncompressedSize() != 0 { now += 10 addMsg(now) } diff --git a/go/mcap/writer.go b/go/mcap/writer.go index d181a643a7..811daab982 100644 --- a/go/mcap/writer.go +++ b/go/mcap/writer.go @@ -8,6 +8,7 @@ import ( "math" "sort" + "github.com/anacrolix/missinggo/v2/panicif" "github.com/klauspost/compress/zstd" "github.com/pierrec/lz4/v4" ) @@ -31,7 +32,7 @@ type Writer struct { w *writeSizer buf []byte msg []byte - uncompressed *bytes.Buffer + uncompressed bytes.Buffer compressed *bytes.Buffer compressedWriter *countingCRCWriter @@ -110,7 +111,7 @@ func (w *Writer) WriteSchema(s *Schema) (err error) { offset += putPrefixedString(w.msg[offset:], s.Encoding) offset += putPrefixedBytes(w.msg[offset:], s.Data) if w.opts.Chunked && !w.closed { - _, err = w.writeRecord(w.compressedWriter, OpSchema, w.msg[:offset]) + _, err = w.writeRecord(w.uncompressedWriter(), OpSchema, w.msg[:offset]) } else { _, err = w.writeRecord(w.w, OpSchema, w.msg[:offset]) } @@ -149,7 +150,7 @@ func (w *Writer) WriteChannel(c *Channel) error { offset += copy(w.msg[offset:], userdata) var err error if w.opts.Chunked && !w.closed { - _, err = w.writeRecord(w.compressedWriter, OpChannel, w.msg[:offset]) + _, err = w.writeRecord(w.uncompressedWriter(), OpChannel, w.msg[:offset]) if err != nil { return err } @@ -193,8 +194,8 @@ func (w *Writer) WriteMessage(m *Message) error { } w.messageIndexes[m.ChannelID] = idx } - idx.Add(m.LogTime, uint64(w.compressedWriter.Size())) - _, err := w.writeRecord(w.compressedWriter, OpMessage, w.msg[:offset]) + idx.Add(m.LogTime, uint64(w.uncompressedSize())) + _, err := w.writeRecord(w.uncompressedWriter(), OpMessage, w.msg[:offset]) if err != nil { return err } @@ -205,7 +206,7 @@ func (w *Writer) WriteMessage(m *Message) error { if m.LogTime < w.currentChunkStartTime { w.currentChunkStartTime = m.LogTime } - if w.compressedWriter.Size() > w.opts.ChunkSize { + if int64(w.uncompressedSize()) > w.opts.ChunkSize { err := w.flushActiveChunk() if err != nil { return err @@ -414,8 +415,13 @@ func (w *Writer) WriteDataEnd(e *DataEnd) error { return err } +func (w *Writer) uncompressedSize() int { + panicif.NotEq(w.compressedWriter.Size(), int64(w.uncompressed.Len())) + return w.uncompressed.Len() +} + func (w *Writer) flushActiveChunk() error { - if w.compressedWriter.Size() == 0 { + if w.uncompressedSize() == 0 { return nil } @@ -424,11 +430,11 @@ func (w *Writer) flushActiveChunk() error { return err } crc := w.compressedWriter.CRC() - compressedlen := w.compressed.Len() - uncompressedlen := w.compressedWriter.Size() + compressedLen := w.compressed.Len() + uncompressedLen := w.uncompressedSize() // the "top fields" are all fields of the chunk record except for the compressed records. topFieldsLen := 8 + 8 + 8 + 4 + 4 + len(w.opts.Compression) + 8 - msglen := topFieldsLen + compressedlen + msgLen := topFieldsLen + compressedLen chunkStartOffset := w.w.Size() var start, end uint64 if w.currentChunkMessageCount != 0 { @@ -446,10 +452,10 @@ func (w *Writer) flushActiveChunk() error { return err } - offset += putUint64(w.msg[offset:], uint64(msglen)) + offset += putUint64(w.msg[offset:], uint64(msgLen)) offset += putUint64(w.msg[offset:], start) offset += putUint64(w.msg[offset:], end) - offset += putUint64(w.msg[offset:], uint64(uncompressedlen)) + offset += putUint64(w.msg[offset:], uint64(uncompressedLen)) offset += putUint32(w.msg[offset:], crc) offset += putPrefixedString(w.msg[offset:], string(w.opts.Compression)) offset += putUint64(w.msg[offset:], uint64(w.compressed.Len())) @@ -465,6 +471,7 @@ func (w *Writer) flushActiveChunk() error { w.compressedWriter.Reset(w.compressed) w.compressedWriter.ResetSize() w.compressedWriter.ResetCRC() + w.uncompressed.Reset() chunkEndOffset := w.w.Size() // message indexes @@ -492,8 +499,8 @@ func (w *Writer) flushActiveChunk() error { MessageIndexOffsets: messageIndexOffsets, MessageIndexLength: messageIndexLength, Compression: w.opts.Compression, - CompressedSize: uint64(compressedlen), - UncompressedSize: uint64(uncompressedlen), + CompressedSize: uint64(compressedLen), + UncompressedSize: uint64(uncompressedLen), }) for _, idx := range w.messageIndexes { idx.Reset() @@ -904,7 +911,6 @@ func NewWriter(w io.Writer, opts *WriterOptions) (*Writer, error) { channels: make(map[uint16]*Channel), schemas: make(map[uint16]*Schema), messageIndexes: make(map[uint16]*MessageIndex), - uncompressed: &bytes.Buffer{}, compressed: &compressed, compressedWriter: compressedWriter, currentChunkStartTime: math.MaxUint64, @@ -918,3 +924,20 @@ func NewWriter(w io.Writer, opts *WriterOptions) (*Writer, error) { opts: opts, }, nil } + +func (w *Writer) assertUncompressedSize() { + panicif.NotEq(w.compressedWriter.Size(), int64(w.uncompressed.Len())) +} + +func (w *Writer) uncompressedWriter() io.Writer { + return io.MultiWriter(&w.uncompressed, w.compressedWriter) +} + +func (w *Writer) writeUncompressed(b []byte) (int, error) { + n1, err := w.uncompressed.Write(b) + panicif.Err(err) + n2, err := w.compressedWriter.Write(b) + panicif.Err(err) + panicif.NotEq(n1, n2) + return n1, nil +} From 2e9a01b5a824f985eb5377980b410e6977fcf165 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Wed, 23 Oct 2024 16:38:03 +1100 Subject: [PATCH 2/9] It all checks out... --- go/mcap/writer.go | 76 +++++++++++++++++++++++------------------------ 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/go/mcap/writer.go b/go/mcap/writer.go index 811daab982..86a14079b0 100644 --- a/go/mcap/writer.go +++ b/go/mcap/writer.go @@ -426,11 +426,18 @@ func (w *Writer) flushActiveChunk() error { } err := w.compressedWriter.Close() - if err != nil { - return err - } - crc := w.compressedWriter.CRC() - compressedLen := w.compressed.Len() + panicif.Err(err) + + var c1 bytes.Buffer + cw, err := w.newCompressedWriter(&c1) + panicif.Err(err) + cw.Write(w.uncompressed.Bytes()) + crc := cw.CRC() + panicif.NotEq(cw.CRC(), w.compressedWriter.CRC()) + cw.Close() + panicif.NotEq(c1.Len(), w.compressed.Len()) + compressedLen := c1.Len() + uncompressedLen := w.uncompressedSize() // the "top fields" are all fields of the chunk record except for the compressed records. topFieldsLen := 8 + 8 + 8 + 4 + 4 + len(w.opts.Compression) + 8 @@ -458,12 +465,12 @@ func (w *Writer) flushActiveChunk() error { offset += putUint64(w.msg[offset:], uint64(uncompressedLen)) offset += putUint32(w.msg[offset:], crc) offset += putPrefixedString(w.msg[offset:], string(w.opts.Compression)) - offset += putUint64(w.msg[offset:], uint64(w.compressed.Len())) + offset += putUint64(w.msg[offset:], uint64(c1.Len())) _, err = w.w.Write(w.msg[:offset]) if err != nil { return err } - _, err = w.w.Write(w.compressed.Bytes()) + _, err = w.w.Write(c1.Bytes()) if err != nil { return err } @@ -863,16 +870,8 @@ func encoderLevelFromZstd(level CompressionLevel) zstd.EncoderLevel { } } -// NewWriter returns a new MCAP writer. -func NewWriter(w io.Writer, opts *WriterOptions) (*Writer, error) { - writer := newWriteSizer(w, opts.IncludeCRC) - if !opts.SkipMagic { - if _, err := writer.Write(Magic); err != nil { - return nil, err - } - } - compressed := bytes.Buffer{} - var compressedWriter *countingCRCWriter +func (w *Writer) newCompressedWriter(compressed *bytes.Buffer) (compressedWriter *countingCRCWriter, err error) { + opts := w.opts if opts.Chunked { switch { case opts.Compressor != nil: // must be top @@ -882,22 +881,22 @@ func NewWriter(w io.Writer, opts *WriterOptions) (*Writer, error) { if opts.Compressor.Compression() == "" { return nil, fmt.Errorf("custom compressor requires compression format") } - opts.Compressor.Compressor().Reset(&compressed) + opts.Compressor.Compressor().Reset(compressed) compressedWriter = newCountingCRCWriter(opts.Compressor.Compressor(), opts.IncludeCRC) case opts.Compression == CompressionZSTD: level := encoderLevelFromZstd(opts.CompressionLevel) - zw, err := zstd.NewWriter(&compressed, zstd.WithEncoderLevel(level)) + zw, err := zstd.NewWriter(compressed, zstd.WithEncoderLevel(level)) if err != nil { return nil, err } compressedWriter = newCountingCRCWriter(zw, opts.IncludeCRC) case opts.Compression == CompressionLZ4: level := encoderLevelFromLZ4(opts.CompressionLevel) - lzw := lz4.NewWriter(&compressed) + lzw := lz4.NewWriter(compressed) _ = lzw.Apply(lz4.CompressionLevelOption(level)) compressedWriter = newCountingCRCWriter(lzw, opts.IncludeCRC) case opts.Compression == CompressionNone: - compressedWriter = newCountingCRCWriter(bufCloser{&compressed}, opts.IncludeCRC) + compressedWriter = newCountingCRCWriter(bufCloser{compressed}, opts.IncludeCRC) default: return nil, fmt.Errorf("unsupported compression") } @@ -905,14 +904,25 @@ func NewWriter(w io.Writer, opts *WriterOptions) (*Writer, error) { opts.ChunkSize = 1024 * 1024 } } - return &Writer{ + return +} + +// NewWriter returns a new MCAP writer. +func NewWriter(w io.Writer, opts *WriterOptions) (ret *Writer, err error) { + writer := newWriteSizer(w, opts.IncludeCRC) + if !opts.SkipMagic { + if _, err := writer.Write(Magic); err != nil { + return nil, err + } + } + // TODO: Check here that compression options are valid? + ret = &Writer{ w: writer, buf: make([]byte, 32), channels: make(map[uint16]*Channel), schemas: make(map[uint16]*Schema), messageIndexes: make(map[uint16]*MessageIndex), - compressed: &compressed, - compressedWriter: compressedWriter, + compressed: new(bytes.Buffer), currentChunkStartTime: math.MaxUint64, currentChunkEndTime: 0, currentChunkMessageCount: 0, @@ -922,22 +932,12 @@ func NewWriter(w io.Writer, opts *WriterOptions) (*Writer, error) { MessageEndTime: 0, }, opts: opts, - }, nil -} - -func (w *Writer) assertUncompressedSize() { - panicif.NotEq(w.compressedWriter.Size(), int64(w.uncompressed.Len())) + } + ret.compressedWriter, err = ret.newCompressedWriter(ret.compressed) + panicif.Err(err) + return ret, nil } func (w *Writer) uncompressedWriter() io.Writer { return io.MultiWriter(&w.uncompressed, w.compressedWriter) } - -func (w *Writer) writeUncompressed(b []byte) (int, error) { - n1, err := w.uncompressed.Write(b) - panicif.Err(err) - n2, err := w.compressedWriter.Write(b) - panicif.Err(err) - panicif.NotEq(n1, n2) - return n1, nil -} From 5fc835aac786ed634a83445232709a936b928164 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Wed, 23 Oct 2024 17:47:31 +1100 Subject: [PATCH 3/9] Make compressedWriter inactive as possible --- go/mcap/writer.go | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/go/mcap/writer.go b/go/mcap/writer.go index 86a14079b0..81ba271d4b 100644 --- a/go/mcap/writer.go +++ b/go/mcap/writer.go @@ -33,7 +33,6 @@ type Writer struct { buf []byte msg []byte uncompressed bytes.Buffer - compressed *bytes.Buffer compressedWriter *countingCRCWriter currentChunkStartTime uint64 @@ -416,7 +415,7 @@ func (w *Writer) WriteDataEnd(e *DataEnd) error { } func (w *Writer) uncompressedSize() int { - panicif.NotEq(w.compressedWriter.Size(), int64(w.uncompressed.Len())) + //panicif.NotEq(w.compressedWriter.Size(), int64(w.uncompressed.Len())) return w.uncompressed.Len() } @@ -425,17 +424,12 @@ func (w *Writer) flushActiveChunk() error { return nil } - err := w.compressedWriter.Close() - panicif.Err(err) - var c1 bytes.Buffer cw, err := w.newCompressedWriter(&c1) panicif.Err(err) cw.Write(w.uncompressed.Bytes()) crc := cw.CRC() - panicif.NotEq(cw.CRC(), w.compressedWriter.CRC()) cw.Close() - panicif.NotEq(c1.Len(), w.compressed.Len()) compressedLen := c1.Len() uncompressedLen := w.uncompressedSize() @@ -474,10 +468,7 @@ func (w *Writer) flushActiveChunk() error { if err != nil { return err } - w.compressed.Reset() - w.compressedWriter.Reset(w.compressed) - w.compressedWriter.ResetSize() - w.compressedWriter.ResetCRC() + cw.Reset(io.Discard) w.uncompressed.Reset() chunkEndOffset := w.w.Size() @@ -922,7 +913,6 @@ func NewWriter(w io.Writer, opts *WriterOptions) (ret *Writer, err error) { channels: make(map[uint16]*Channel), schemas: make(map[uint16]*Schema), messageIndexes: make(map[uint16]*MessageIndex), - compressed: new(bytes.Buffer), currentChunkStartTime: math.MaxUint64, currentChunkEndTime: 0, currentChunkMessageCount: 0, @@ -933,11 +923,14 @@ func NewWriter(w io.Writer, opts *WriterOptions) (ret *Writer, err error) { }, opts: opts, } - ret.compressedWriter, err = ret.newCompressedWriter(ret.compressed) + ret.compressedWriter, err = ret.newCompressedWriter(nil) panicif.Err(err) return ret, nil } func (w *Writer) uncompressedWriter() io.Writer { - return io.MultiWriter(&w.uncompressed, w.compressedWriter) + return &w.uncompressed + //return io.MultiWriter(&w.uncompressed, w.compressedWriter) } + +const checkWriter = false From da15e5762ab9498c3d1b6bb1e79f7833bb691fd8 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Thu, 24 Oct 2024 10:34:35 +1100 Subject: [PATCH 4/9] Finally figured it out --- go/mcap/writer.go | 97 ++++++++++++++++++++++++----------------------- 1 file changed, 49 insertions(+), 48 deletions(-) diff --git a/go/mcap/writer.go b/go/mcap/writer.go index 81ba271d4b..7638abe6e3 100644 --- a/go/mcap/writer.go +++ b/go/mcap/writer.go @@ -24,16 +24,15 @@ type Writer struct { // MetadataIndexes created over the course of the recording. MetadataIndexes []*MetadataIndex - channelIDs []uint16 - schemaIDs []uint16 - channels map[uint16]*Channel - schemas map[uint16]*Schema - messageIndexes map[uint16]*MessageIndex - w *writeSizer - buf []byte - msg []byte - uncompressed bytes.Buffer - compressedWriter *countingCRCWriter + channelIDs []uint16 + schemaIDs []uint16 + channels map[uint16]*Channel + schemas map[uint16]*Schema + messageIndexes map[uint16]*MessageIndex + w *writeSizer + buf []byte + msg []byte + uncompressed bytes.Buffer currentChunkStartTime uint64 currentChunkEndTime uint64 @@ -420,6 +419,8 @@ func (w *Writer) uncompressedSize() int { } func (w *Writer) flushActiveChunk() error { + opts := w.opts + fmt.Printf("chunked: %v, compressor: %v, compression: %v\n", opts.Chunked, opts.Compressor != nil, opts.Compression) if w.uncompressedSize() == 0 { return nil } @@ -427,9 +428,12 @@ func (w *Writer) flushActiveChunk() error { var c1 bytes.Buffer cw, err := w.newCompressedWriter(&c1) panicif.Err(err) - cw.Write(w.uncompressed.Bytes()) + n, err := cw.Write(w.uncompressed.Bytes()) + panicif.Err(err) + panicif.NotEq(n, w.uncompressed.Len()) crc := cw.CRC() - cw.Close() + err = cw.Close() + panicif.Err(err) compressedLen := c1.Len() uncompressedLen := w.uncompressedSize() @@ -468,7 +472,6 @@ func (w *Writer) flushActiveChunk() error { if err != nil { return err } - cw.Reset(io.Discard) w.uncompressed.Reset() chunkEndOffset := w.w.Size() @@ -863,37 +866,36 @@ func encoderLevelFromZstd(level CompressionLevel) zstd.EncoderLevel { func (w *Writer) newCompressedWriter(compressed *bytes.Buffer) (compressedWriter *countingCRCWriter, err error) { opts := w.opts - if opts.Chunked { - switch { - case opts.Compressor != nil: // must be top - // override the compression option. We can't check for a mismatch here - // because "none compression" is an empty string. - opts.Compression = opts.Compressor.Compression() - if opts.Compressor.Compression() == "" { - return nil, fmt.Errorf("custom compressor requires compression format") - } - opts.Compressor.Compressor().Reset(compressed) - compressedWriter = newCountingCRCWriter(opts.Compressor.Compressor(), opts.IncludeCRC) - case opts.Compression == CompressionZSTD: - level := encoderLevelFromZstd(opts.CompressionLevel) - zw, err := zstd.NewWriter(compressed, zstd.WithEncoderLevel(level)) - if err != nil { - return nil, err - } - compressedWriter = newCountingCRCWriter(zw, opts.IncludeCRC) - case opts.Compression == CompressionLZ4: - level := encoderLevelFromLZ4(opts.CompressionLevel) - lzw := lz4.NewWriter(compressed) - _ = lzw.Apply(lz4.CompressionLevelOption(level)) - compressedWriter = newCountingCRCWriter(lzw, opts.IncludeCRC) - case opts.Compression == CompressionNone: - compressedWriter = newCountingCRCWriter(bufCloser{compressed}, opts.IncludeCRC) - default: - return nil, fmt.Errorf("unsupported compression") + if !opts.Chunked { + panic("this should never get called when chunking isn't enabled") + } + switch { + case opts.Compressor != nil: // must be top + // override the compression option. We can't check for a mismatch here + // because "none compression" is an empty string. + opts.Compression = opts.Compressor.Compression() + if opts.Compressor.Compression() == "" { + return nil, fmt.Errorf("custom compressor requires compression format") } - if opts.ChunkSize == 0 { - opts.ChunkSize = 1024 * 1024 + c := opts.Compressor.NewCompressor() + c.Reset(compressed) + compressedWriter = newCountingCRCWriter(c, opts.IncludeCRC) + case opts.Compression == CompressionZSTD: + level := encoderLevelFromZstd(opts.CompressionLevel) + zw, err := zstd.NewWriter(compressed, zstd.WithEncoderLevel(level)) + if err != nil { + return nil, err } + compressedWriter = newCountingCRCWriter(zw, opts.IncludeCRC) + case opts.Compression == CompressionLZ4: + level := encoderLevelFromLZ4(opts.CompressionLevel) + lzw := lz4.NewWriter(compressed) + _ = lzw.Apply(lz4.CompressionLevelOption(level)) + compressedWriter = newCountingCRCWriter(lzw, opts.IncludeCRC) + case opts.Compression == CompressionNone: + compressedWriter = newCountingCRCWriter(bufCloser{compressed}, opts.IncludeCRC) + default: + return nil, fmt.Errorf("unsupported compression") } return } @@ -906,7 +908,11 @@ func NewWriter(w io.Writer, opts *WriterOptions) (ret *Writer, err error) { return nil, err } } - // TODO: Check here that compression options are valid? + if opts.ChunkSize == 0 { + opts.ChunkSize = 1024 * 1024 + } + // Should we check here that compression options are valid? For now I haven't added it as it's + // going to come up pretty quickly while writing anyway, and introduces some extra complexity. ret = &Writer{ w: writer, buf: make([]byte, 32), @@ -923,14 +929,9 @@ func NewWriter(w io.Writer, opts *WriterOptions) (ret *Writer, err error) { }, opts: opts, } - ret.compressedWriter, err = ret.newCompressedWriter(nil) - panicif.Err(err) return ret, nil } func (w *Writer) uncompressedWriter() io.Writer { return &w.uncompressed - //return io.MultiWriter(&w.uncompressed, w.compressedWriter) } - -const checkWriter = false From b5141c9fbd721651b0359c0a3c200e6f181dedd9 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Thu, 24 Oct 2024 17:15:12 +1100 Subject: [PATCH 5/9] Modify CustomCompressor interface to create new compressors --- go/mcap/writer.go | 9 ++++++--- go/mcap/writer_test.go | 16 +++++++++------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/go/mcap/writer.go b/go/mcap/writer.go index 7638abe6e3..7305ca61b9 100644 --- a/go/mcap/writer.go +++ b/go/mcap/writer.go @@ -752,13 +752,16 @@ const ( ) type CustomCompressor interface { - Compressor() ResettableWriteCloser + NewCompressor() ResettableWriteCloser Compression() CompressionFormat } // NewCustomCompressor returns a structure that may be supplied to writer // options as a custom chunk compressor. -func NewCustomCompressor(compression CompressionFormat, compressor ResettableWriteCloser) CustomCompressor { +func NewCustomCompressor( + compression CompressionFormat, + compressor func() (ResettableWriteCloser, error), +) CustomCompressor { return &customCompressor{ compression: compression, compressor: compressor, @@ -767,7 +770,7 @@ func NewCustomCompressor(compression CompressionFormat, compressor ResettableWri type customCompressor struct { compression CompressionFormat - compressor ResettableWriteCloser + compressor func() (ResettableWriteCloser, error) } func (c *customCompressor) Compressor() ResettableWriteCloser { diff --git a/go/mcap/writer_test.go b/go/mcap/writer_test.go index f562271159..c121bbed88 100644 --- a/go/mcap/writer_test.go +++ b/go/mcap/writer_test.go @@ -703,16 +703,18 @@ func assertReadable(t *testing.T, rs io.ReadSeeker) { func TestBYOCompressor(t *testing.T) { buf := &bytes.Buffer{} // example - custom lz4 settings - lzw := lz4.NewWriter(nil) blockCount := 0 - require.NoError(t, lzw.Apply(lz4.OnBlockDoneOption(func(int) { - blockCount++ - }))) writer, err := NewWriter(buf, &WriterOptions{ - Chunked: true, - ChunkSize: 1024, - Compressor: NewCustomCompressor("lz4", lzw), + Chunked: true, + ChunkSize: 1024, + Compressor: NewCustomCompressor("lz4", func() (ResettableWriteCloser, error) { + lzw := lz4.NewWriter(nil) + require.NoError(t, lzw.Apply(lz4.OnBlockDoneOption(func(int) { + blockCount++ + }))) + return lzw, nil + }), }) require.NoError(t, err) From be01a5904a9b985d7e988762efdb4bc07f7e4427 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Thu, 24 Oct 2024 17:15:31 +1100 Subject: [PATCH 6/9] Remove chunk compressor opt debugging --- go/mcap/writer.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/go/mcap/writer.go b/go/mcap/writer.go index 7305ca61b9..3eb4396b5e 100644 --- a/go/mcap/writer.go +++ b/go/mcap/writer.go @@ -419,8 +419,6 @@ func (w *Writer) uncompressedSize() int { } func (w *Writer) flushActiveChunk() error { - opts := w.opts - fmt.Printf("chunked: %v, compressor: %v, compression: %v\n", opts.Chunked, opts.Compressor != nil, opts.Compression) if w.uncompressedSize() == 0 { return nil } From d196769becdf44a651494d11c7173330174a106a Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Thu, 24 Oct 2024 17:28:17 +1100 Subject: [PATCH 7/9] Remove panicif, tidy up compressor setup --- go/mcap/writer.go | 74 ++++++++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 33 deletions(-) diff --git a/go/mcap/writer.go b/go/mcap/writer.go index 3eb4396b5e..06cee1a255 100644 --- a/go/mcap/writer.go +++ b/go/mcap/writer.go @@ -8,7 +8,6 @@ import ( "math" "sort" - "github.com/anacrolix/missinggo/v2/panicif" "github.com/klauspost/compress/zstd" "github.com/pierrec/lz4/v4" ) @@ -414,7 +413,6 @@ func (w *Writer) WriteDataEnd(e *DataEnd) error { } func (w *Writer) uncompressedSize() int { - //panicif.NotEq(w.compressedWriter.Size(), int64(w.uncompressed.Len())) return w.uncompressed.Len() } @@ -423,17 +421,21 @@ func (w *Writer) flushActiveChunk() error { return nil } - var c1 bytes.Buffer - cw, err := w.newCompressedWriter(&c1) - panicif.Err(err) - n, err := cw.Write(w.uncompressed.Bytes()) - panicif.Err(err) - panicif.NotEq(n, w.uncompressed.Len()) - crc := cw.CRC() + var buf bytes.Buffer + cw, err := w.newCompressedWriter(&buf) + if err != nil { + return fmt.Errorf("creating compressed writer: %w", err) + } + _, err = cw.Write(w.uncompressed.Bytes()) + if err != nil { + return fmt.Errorf("writing uncompressed data to compressor: %w", err) + } err = cw.Close() - panicif.Err(err) - compressedLen := c1.Len() + if err != nil { + return fmt.Errorf("finalizing compressor: %w", err) + } + compressedLen := buf.Len() uncompressedLen := w.uncompressedSize() // the "top fields" are all fields of the chunk record except for the compressed records. topFieldsLen := 8 + 8 + 8 + 4 + 4 + len(w.opts.Compression) + 8 @@ -459,14 +461,14 @@ func (w *Writer) flushActiveChunk() error { offset += putUint64(w.msg[offset:], start) offset += putUint64(w.msg[offset:], end) offset += putUint64(w.msg[offset:], uint64(uncompressedLen)) - offset += putUint32(w.msg[offset:], crc) + offset += putUint32(w.msg[offset:], cw.CRC()) offset += putPrefixedString(w.msg[offset:], string(w.opts.Compression)) - offset += putUint64(w.msg[offset:], uint64(c1.Len())) + offset += putUint64(w.msg[offset:], uint64(buf.Len())) _, err = w.w.Write(w.msg[:offset]) if err != nil { return err } - _, err = w.w.Write(c1.Bytes()) + _, err = w.w.Write(buf.Bytes()) if err != nil { return err } @@ -750,7 +752,7 @@ const ( ) type CustomCompressor interface { - NewCompressor() ResettableWriteCloser + NewCompressor() (ResettableWriteCloser, error) Compression() CompressionFormat } @@ -771,8 +773,8 @@ type customCompressor struct { compressor func() (ResettableWriteCloser, error) } -func (c *customCompressor) Compressor() ResettableWriteCloser { - return c.compressor +func (c *customCompressor) NewCompressor() (ResettableWriteCloser, error) { + return c.compressor() } func (c *customCompressor) Compression() CompressionFormat { @@ -865,11 +867,7 @@ func encoderLevelFromZstd(level CompressionLevel) zstd.EncoderLevel { } } -func (w *Writer) newCompressedWriter(compressed *bytes.Buffer) (compressedWriter *countingCRCWriter, err error) { - opts := w.opts - if !opts.Chunked { - panic("this should never get called when chunking isn't enabled") - } +func newCompressor(compressed *bytes.Buffer, opts *WriterOptions) (ResettableWriteCloser, error) { switch { case opts.Compressor != nil: // must be top // override the compression option. We can't check for a mismatch here @@ -878,27 +876,37 @@ func (w *Writer) newCompressedWriter(compressed *bytes.Buffer) (compressedWriter if opts.Compressor.Compression() == "" { return nil, fmt.Errorf("custom compressor requires compression format") } - c := opts.Compressor.NewCompressor() - c.Reset(compressed) - compressedWriter = newCountingCRCWriter(c, opts.IncludeCRC) - case opts.Compression == CompressionZSTD: - level := encoderLevelFromZstd(opts.CompressionLevel) - zw, err := zstd.NewWriter(compressed, zstd.WithEncoderLevel(level)) + c, err := opts.Compressor.NewCompressor() if err != nil { return nil, err } - compressedWriter = newCountingCRCWriter(zw, opts.IncludeCRC) + c.Reset(compressed) + return c, nil + case opts.Compression == CompressionZSTD: + level := encoderLevelFromZstd(opts.CompressionLevel) + return zstd.NewWriter(compressed, zstd.WithEncoderLevel(level)) case opts.Compression == CompressionLZ4: level := encoderLevelFromLZ4(opts.CompressionLevel) lzw := lz4.NewWriter(compressed) _ = lzw.Apply(lz4.CompressionLevelOption(level)) - compressedWriter = newCountingCRCWriter(lzw, opts.IncludeCRC) + return lzw, nil case opts.Compression == CompressionNone: - compressedWriter = newCountingCRCWriter(bufCloser{compressed}, opts.IncludeCRC) + return newCountingCRCWriter(bufCloser{compressed}, opts.IncludeCRC), nil default: - return nil, fmt.Errorf("unsupported compression") + return nil, fmt.Errorf("unsupported compression: %v", opts.Compression) + } +} + +func (w *Writer) newCompressedWriter(compressed *bytes.Buffer) (*countingCRCWriter, error) { + opts := w.opts + if !opts.Chunked { + panic("this should never get called when chunking isn't enabled") + } + compressor, err := newCompressor(compressed, opts) + if err != nil { + return nil, err } - return + return newCountingCRCWriter(compressor, opts.IncludeCRC), nil } // NewWriter returns a new MCAP writer. From 2be67dbeff95894d4dae3df21ee47a697721e4ba Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Thu, 24 Oct 2024 17:47:21 +1100 Subject: [PATCH 8/9] go mod tidy --- go/go.work.sum | 7 +++++-- go/mcap/go.mod | 4 ---- go/mcap/go.sum | 19 ++----------------- 3 files changed, 7 insertions(+), 23 deletions(-) diff --git a/go/go.work.sum b/go/go.work.sum index 064edb2057..58d350f79a 100644 --- a/go/go.work.sum +++ b/go/go.work.sum @@ -93,7 +93,7 @@ github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfE github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcMpsXCp3lJ03zYT1PkRd3kQGPn9GVg= github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -122,7 +122,7 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= @@ -153,10 +153,12 @@ go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA= golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 h1:LQmS1nU0twXLA96Kt7U9qtHJEbBk3z6Q0V4UXjZkpr4= golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= @@ -165,6 +167,7 @@ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDE golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= +golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 h1:0c3L82FDQ5rt1bjTBlchS8t6RQ6299/+5bWMnRLh+uI= golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= diff --git a/go/mcap/go.mod b/go/mcap/go.mod index 85505ede63..a1ca83250e 100644 --- a/go/mcap/go.mod +++ b/go/mcap/go.mod @@ -3,19 +3,15 @@ module github.com/foxglove/mcap/go/mcap go 1.23 require ( - github.com/anacrolix/missinggo/v2 v2.8.0 github.com/klauspost/compress v1.16.7 github.com/pierrec/lz4/v4 v4.1.21 github.com/stretchr/testify v1.9.0 ) require ( - github.com/anacrolix/generics v0.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/huandu/xstrings v1.3.2 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 // indirect gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go/mcap/go.sum b/go/mcap/go.sum index ee8e9c76bc..6c07f6f9c1 100644 --- a/go/mcap/go.sum +++ b/go/mcap/go.sum @@ -1,20 +1,10 @@ -github.com/anacrolix/generics v0.0.2 h1:UbtD+KntUGxeGYMC4RwhsETieL9ixGdSptJQRhdy7No= -github.com/anacrolix/generics v0.0.2/go.mod h1:ff2rHB/joTV03aMSSn/AZNnaIpUw0h3njetGsaXcMy8= -github.com/anacrolix/missinggo/v2 v2.8.0 h1:6pGnVOlR6TWL9JM5Msyezij8YHU3+oHO7r82Eql/kpA= -github.com/anacrolix/missinggo/v2 v2.8.0/go.mod h1:vVO5FEziQm+NFmJesc7StpkquZk+WJFCaL0Wp//2sa0= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= -github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= @@ -22,17 +12,12 @@ github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFu github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -golang.org/x/exp v0.0.0-20220428152302-39d4317da171 h1:TfdoLivD44QwvssI9Sv1xwa5DcL5XQr4au4sZ2F2NV4= -golang.org/x/exp v0.0.0-20220428152302-39d4317da171/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= -golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 3e80aa1d59b02cddb67034865f648f6b62db8547 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Thu, 24 Oct 2024 17:48:19 +1100 Subject: [PATCH 9/9] Revert to go1.18 --- go/mcap/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/mcap/go.mod b/go/mcap/go.mod index a1ca83250e..6aa90f8838 100644 --- a/go/mcap/go.mod +++ b/go/mcap/go.mod @@ -1,6 +1,6 @@ module github.com/foxglove/mcap/go/mcap -go 1.23 +go 1.18 require ( github.com/klauspost/compress v1.16.7