From a82efe5fd49963fec07763a2c5274517492a92b1 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Mon, 17 Mar 2025 20:41:36 +0000 Subject: [PATCH 01/57] chore: setup dependabot --- .github/dependabot.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..19a064f --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + # Check for updates to GitHub Actions every week + interval: "weekly" \ No newline at end of file From 9e66d89be8ccdce715c5722cedaa5a37704d952e Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Mon, 17 Mar 2025 20:41:48 +0000 Subject: [PATCH 02/57] refactor: replace with sha --- .github/workflows/codspeed.yaml | 6 +++--- .github/workflows/coverage.yaml | 6 +++--- .github/workflows/pull_request.yaml | 12 ++++++------ .github/workflows/release.yaml | 4 ++-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/codspeed.yaml b/.github/workflows/codspeed.yaml index e052573..e055329 100644 --- a/.github/workflows/codspeed.yaml +++ b/.github/workflows/codspeed.yaml @@ -30,10 +30,10 @@ jobs: name: Run benchmarks runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Setup rust toolchain, cache and cargo-codspeed binary - uses: moonrepo/setup-rust@v1 + uses: moonrepo/setup-rust@ede6de059f8046a5e236c94046823e2af11ca670 # v1.2.2 with: cache-target: release bins: cargo-codspeed @@ -42,7 +42,7 @@ jobs: run: cargo codspeed build - name: Run the benchmarks - uses: CodSpeedHQ/action@v3 + uses: CodSpeedHQ/action@0010eb0ca6e89b80c88e8edaaa07cfe5f3e6664d # v3.5.0 with: run: cargo codspeed run token: ${{ secrets.CODSPEED_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index a6ec5a6..6722517 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -24,9 +24,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout PR branch - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: moonrepo/setup-rust@v1 + uses: moonrepo/setup-rust@ede6de059f8046a5e236c94046823e2af11ca670 # v1.2.2 with: cache-base: main components: llvm-tools-preview @@ -34,7 +34,7 @@ jobs: - name: Run test coverage run: | cargo llvm-cov --all-features --workspace --lcov --output-path lcov.info - - uses: codecov/codecov-action@v5 + - uses: codecov/codecov-action@0565863a31f2c772f9f0395002a31e3f06189574 # v5.4.0 with: files: ./lcov.info env: diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index d99057e..33f413b 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -24,9 +24,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout PR branch - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: moonrepo/setup-rust@v1 + uses: moonrepo/setup-rust@ede6de059f8046a5e236c94046823e2af11ca670 # v1.2.2 with: components: rustfmt cache-base: main @@ -38,9 +38,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout PR branch - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: moonrepo/setup-rust@v1 + uses: moonrepo/setup-rust@ede6de059f8046a5e236c94046823e2af11ca670 # v1.2.2 with: components: clippy cache-base: main @@ -55,9 +55,9 @@ jobs: os: [ubuntu-latest, windows-latest] steps: - name: Checkout PR branch - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: moonrepo/setup-rust@v1 + uses: moonrepo/setup-rust@ede6de059f8046a5e236c94046823e2af11ca670 # v1.2.2 with: cache-base: main - name: Run test diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 60e86c5..c95d06c 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -12,9 +12,9 @@ jobs: steps: - name: Checkout branch - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Rust - uses: moonrepo/setup-rust@v1 + uses: moonrepo/setup-rust@ede6de059f8046a5e236c94046823e2af11ca670 # v1.2.2 - name: Publish crates.io run: | cargo publish -p messagepack-core From 59748ba8247d3872f4c36ed3d58041ecc01d5e2b Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Mon, 17 Mar 2025 20:45:16 +0000 Subject: [PATCH 03/57] refactor: run only rust file changed --- .github/workflows/coverage.yaml | 7 +++++++ .github/workflows/pull_request.yaml | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index 6722517..bf0a76c 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -6,9 +6,16 @@ on: - opened - reopened - synchronize + paths: + - "**.rs" + - '**Cargo.toml' + - '**Cargo.lock' + - "rust-toolchain.toml" + - "rustfmt.toml" push: branches: - main + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 33f413b..48063aa 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -6,9 +6,16 @@ on: - opened - reopened - synchronize + paths: + - "**.rs" + - '**Cargo.toml' + - '**Cargo.lock' + - "rust-toolchain.toml" + - "rustfmt.toml" push: branches: - main + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} From 99f2891cf0bc8e611a823acde8e05e6898c23359 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Jul 2025 02:20:15 +0000 Subject: [PATCH 04/57] chore(deps): bump CodSpeedHQ/action from 3.5.0 to 3.6.1 Bumps [CodSpeedHQ/action](https://github.com/codspeedhq/action) from 3.5.0 to 3.6.1. - [Release notes](https://github.com/codspeedhq/action/releases) - [Changelog](https://github.com/CodSpeedHQ/action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codspeedhq/action/compare/0010eb0ca6e89b80c88e8edaaa07cfe5f3e6664d...ad71f92d9429aabfcf70738be9defdbbfc7b75e2) --- updated-dependencies: - dependency-name: CodSpeedHQ/action dependency-version: 3.6.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/codspeed.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codspeed.yaml b/.github/workflows/codspeed.yaml index e055329..6677d44 100644 --- a/.github/workflows/codspeed.yaml +++ b/.github/workflows/codspeed.yaml @@ -42,7 +42,7 @@ jobs: run: cargo codspeed build - name: Run the benchmarks - uses: CodSpeedHQ/action@0010eb0ca6e89b80c88e8edaaa07cfe5f3e6664d # v3.5.0 + uses: CodSpeedHQ/action@ad71f92d9429aabfcf70738be9defdbbfc7b75e2 # v3.6.1 with: run: cargo codspeed run token: ${{ secrets.CODSPEED_TOKEN }} \ No newline at end of file From b3106a49152438fd58edd7627abe410babec1a1f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 12 Jul 2025 09:32:48 +0000 Subject: [PATCH 05/57] chore(deps): bump codecov/codecov-action from 5.4.0 to 5.4.3 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 5.4.0 to 5.4.3. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/0565863a31f2c772f9f0395002a31e3f06189574...18283e04ce6e62d37312384ff67231eb8fd56d24) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-version: 5.4.3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/coverage.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index bf0a76c..3252e9a 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -41,7 +41,7 @@ jobs: - name: Run test coverage run: | cargo llvm-cov --all-features --workspace --lcov --output-path lcov.info - - uses: codecov/codecov-action@0565863a31f2c772f9f0395002a31e3f06189574 # v5.4.0 + - uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5.4.3 with: files: ./lcov.info env: From 1eb0ed48ee13b0a4ad626189f7c384f6df32f884 Mon Sep 17 00:00:00 2001 From: tunamaguro <79092292+tunamaguro@users.noreply.github.com> Date: Sun, 13 Jul 2025 16:11:59 +0900 Subject: [PATCH 06/57] Potential fix for code scanning alert no. 6: Workflow does not contain permissions Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .github/workflows/pull_request.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 48063aa..50c7c18 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -1,5 +1,8 @@ name: Rust CI +permissions: + contents: read + on: pull_request: types: From 0bb2de960f919f75aac392560f9d559d08098671 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Thu, 4 Sep 2025 20:13:16 +0000 Subject: [PATCH 07/57] chore: bump rust version --- rust-toolchain.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 7e2c45e..f54446d 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,5 +1,5 @@ [toolchain] -channel = "1.85.0" +channel = "1.89.0" components = ["rustc", "cargo", "rustfmt", "clippy"] profile = "minimal" targets = [] From 3cb4725c9a2f30f1fdae49c89df2308f75784235 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Thu, 4 Sep 2025 20:59:19 +0000 Subject: [PATCH 08/57] feat: timestamp --- messagepack-core/src/decode/mod.rs | 1 + messagepack-core/src/decode/timestamp.rs | 88 +++++++++++++++++++ messagepack-core/src/encode/extension.rs | 9 +- messagepack-core/src/encode/mod.rs | 1 + messagepack-core/src/encode/timestamp.rs | 26 ++++++ messagepack-core/src/formats.rs | 4 - messagepack-core/src/lib.rs | 1 + messagepack-core/src/timestamp.rs | 105 +++++++++++++++++++++++ 8 files changed, 227 insertions(+), 8 deletions(-) create mode 100644 messagepack-core/src/decode/timestamp.rs create mode 100644 messagepack-core/src/encode/timestamp.rs create mode 100644 messagepack-core/src/timestamp.rs diff --git a/messagepack-core/src/decode/mod.rs b/messagepack-core/src/decode/mod.rs index 7da2ae3..a02660f 100644 --- a/messagepack-core/src/decode/mod.rs +++ b/messagepack-core/src/decode/mod.rs @@ -15,6 +15,7 @@ mod nil; pub use nil::NilDecoder; mod str; pub use str::StrDecoder; +mod timestamp; /// Messagepack Encode Error #[derive(Debug, Copy, Clone, PartialOrd, Ord, PartialEq, Eq)] diff --git a/messagepack-core/src/decode/timestamp.rs b/messagepack-core/src/decode/timestamp.rs new file mode 100644 index 0000000..12db2e2 --- /dev/null +++ b/messagepack-core/src/decode/timestamp.rs @@ -0,0 +1,88 @@ +use super::{Decode, Error, NbyteReader, Result}; +use crate::{ + Format, + timestamp::{TIMESTAMP_EXTENSION_TYPE, Timestamp32, Timestamp64, Timestamp96}, +}; + +impl<'a> Decode<'a> for Timestamp32 { + type Value = Timestamp32; + fn decode(buf: &'a [u8]) -> Result<(Self::Value, &'a [u8])> { + let (format, buf) = Format::decode(buf)?; + match format { + Format::FixExt4 => Self::decode_with_format(format, buf), + _ => Err(Error::UnexpectedFormat), + } + } + fn decode_with_format(format: crate::Format, buf: &'a [u8]) -> Result<(Self::Value, &'a [u8])> { + let (len, buf) = match format { + Format::FixExt4 => (4, buf), + _ => return Err(Error::UnexpectedFormat), + }; + let (ext_type, buf) = buf.split_first().ok_or(Error::EofData)?; + let ext_type = (*ext_type) as i8; + if ext_type != TIMESTAMP_EXTENSION_TYPE { + return Err(Error::InvalidData); + } + + let (data, rest) = buf.split_at_checked(len).ok_or(Error::EofData)?; + let timestamp = Self::from_buf(data.try_into().expect("expect 4 len")); + Ok((timestamp, rest)) + } +} + +impl<'a> Decode<'a> for Timestamp64 { + type Value = Timestamp64; + fn decode(buf: &'a [u8]) -> Result<(Self::Value, &'a [u8])> { + let (format, buf) = Format::decode(buf)?; + match format { + Format::FixExt8 => Self::decode_with_format(format, buf), + _ => Err(Error::UnexpectedFormat), + } + } + fn decode_with_format(format: crate::Format, buf: &'a [u8]) -> Result<(Self::Value, &'a [u8])> { + let (len, buf) = match format { + Format::FixExt8 => (8, buf), + _ => return Err(Error::UnexpectedFormat), + }; + let (ext_type, buf) = buf.split_first().ok_or(Error::EofData)?; + let ext_type = (*ext_type) as i8; + if ext_type != TIMESTAMP_EXTENSION_TYPE { + return Err(Error::InvalidData); + } + + let (data, rest) = buf.split_at_checked(len).ok_or(Error::EofData)?; + let timestamp = Self::from_buf(data.try_into().expect("expect 8 len")); + Ok((timestamp, rest)) + } +} + +impl<'a> Decode<'a> for Timestamp96 { + type Value = Timestamp96; + fn decode(buf: &'a [u8]) -> Result<(Self::Value, &'a [u8])> { + let (format, buf) = Format::decode(buf)?; + match format { + Format::Ext8 => Self::decode_with_format(format, buf), + _ => Err(Error::UnexpectedFormat), + } + } + fn decode_with_format(format: crate::Format, buf: &'a [u8]) -> Result<(Self::Value, &'a [u8])> { + let (len, buf) = match format { + Format::Ext8 => NbyteReader::<1>::read(buf)?, + _ => return Err(Error::UnexpectedFormat), + }; + const TIMESTAMP64_DATA_LENGTH: usize = 12; + if len != TIMESTAMP64_DATA_LENGTH { + return Err(Error::InvalidData); + } + + let (ext_type, buf) = buf.split_first().ok_or(Error::EofData)?; + let ext_type = (*ext_type) as i8; + if ext_type != TIMESTAMP_EXTENSION_TYPE { + return Err(Error::InvalidData); + } + + let (data, rest) = buf.split_at_checked(len).ok_or(Error::EofData)?; + let timestamp = Self::from_buf(data.try_into().expect("expect 12 len")); + Ok((timestamp, rest)) + } +} diff --git a/messagepack-core/src/encode/extension.rs b/messagepack-core/src/encode/extension.rs index c0484bd..5ba0476 100644 --- a/messagepack-core/src/encode/extension.rs +++ b/messagepack-core/src/encode/extension.rs @@ -1,8 +1,9 @@ use super::{Encode, Error, Result}; -use crate::{ - formats::{Format, U8_MAX, U16_MAX, U32_MAX}, - io::IoWrite, -}; +use crate::{formats::Format, io::IoWrite}; + +pub const U8_MAX: usize = u8::MAX as usize; +pub const U16_MAX: usize = u16::MAX as usize; +pub const U32_MAX: usize = u32::MAX as usize; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] pub struct ExtensionEncoder<'data> { diff --git a/messagepack-core/src/encode/mod.rs b/messagepack-core/src/encode/mod.rs index 4a56498..2221394 100644 --- a/messagepack-core/src/encode/mod.rs +++ b/messagepack-core/src/encode/mod.rs @@ -7,6 +7,7 @@ pub mod int; pub mod map; pub mod nil; pub mod str; +mod timestamp; pub use array::{ArrayDataEncoder, ArrayEncoder, ArrayFormatEncoder}; pub use bin::BinaryEncoder; diff --git a/messagepack-core/src/encode/timestamp.rs b/messagepack-core/src/encode/timestamp.rs new file mode 100644 index 0000000..6cce462 --- /dev/null +++ b/messagepack-core/src/encode/timestamp.rs @@ -0,0 +1,26 @@ +use super::{Encode, Result, extension::ExtensionEncoder}; +use crate::{ + io::IoWrite, + timestamp::{TIMESTAMP_EXTENSION_TYPE, Timestamp32, Timestamp64, Timestamp96}, +}; + +impl Encode for Timestamp32 { + fn encode(&self, writer: &mut W) -> Result { + let buf = self.to_buf(); + ExtensionEncoder::new(TIMESTAMP_EXTENSION_TYPE, &buf).encode(writer) + } +} + +impl Encode for Timestamp64 { + fn encode(&self, writer: &mut W) -> Result { + let buf = self.to_buf(); + ExtensionEncoder::new(TIMESTAMP_EXTENSION_TYPE, &buf).encode(writer) + } +} + +impl Encode for Timestamp96 { + fn encode(&self, writer: &mut W) -> Result { + let buf = self.to_buf(); + ExtensionEncoder::new(TIMESTAMP_EXTENSION_TYPE, &buf).encode(writer) + } +} diff --git a/messagepack-core/src/formats.rs b/messagepack-core/src/formats.rs index ae0d51c..bc3765f 100644 --- a/messagepack-core/src/formats.rs +++ b/messagepack-core/src/formats.rs @@ -40,10 +40,6 @@ const ARRAY32: u8 = 0xdd; const MAP16: u8 = 0xde; const MAP32: u8 = 0xdf; -pub const U8_MAX: usize = u8::MAX as usize; -pub const U16_MAX: usize = u16::MAX as usize; -pub const U32_MAX: usize = u32::MAX as usize; - #[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)] pub enum Format { PositiveFixInt(u8), diff --git a/messagepack-core/src/lib.rs b/messagepack-core/src/lib.rs index 70d23b7..9df132b 100644 --- a/messagepack-core/src/lib.rs +++ b/messagepack-core/src/lib.rs @@ -7,6 +7,7 @@ pub mod decode; pub mod encode; mod formats; pub mod io; +pub mod timestamp; pub use decode::Decode; pub use encode::Encode; diff --git a/messagepack-core/src/timestamp.rs b/messagepack-core/src/timestamp.rs new file mode 100644 index 0000000..bc90a8b --- /dev/null +++ b/messagepack-core/src/timestamp.rs @@ -0,0 +1,105 @@ +pub(crate) const TIMESTAMP_EXTENSION_TYPE: i8 = -1; + +/// Represents timestamp 32 extension type. +/// This stores 32bit unsigned seconds +pub struct Timestamp32 { + secs: u32, +} + +impl Timestamp32 { + pub fn new(seconds: u32) -> Self { + Self { secs: seconds } + } + + pub fn seconds(&self) -> u32 { + self.secs + } + + pub(crate) fn to_buf(&self) -> [u8; 4] { + self.secs.to_be_bytes() + } + + pub(crate) fn from_buf(buf: [u8; 4]) -> Self { + Self { + secs: u32::from_be_bytes(buf), + } + } +} + +/// Represents timestamp 64 extension type. +/// This stores 34bit unsigned seconds and 30bit nanoseconds +pub struct Timestamp64 { + data: [u8; 8], +} + +impl Timestamp64 { + pub fn nanos(&self) -> u32 { + let mut buf = [0u8; 4]; + buf.copy_from_slice(&self.data[..4]); + let nanosec = u32::from_be_bytes(buf); + nanosec >> 2 + } + + pub fn seconds(&self) -> u64 { + // 34bit mask + const MASK: u64 = 0x3FFFF; + let mut buf = [0u8; 8]; + buf.copy_from_slice(&self.data[..]); + let seconds = u64::from_be_bytes(buf); + + seconds & MASK + } + + pub(crate) fn to_buf(&self) -> [u8; 8] { + self.data + } + + pub(crate) fn from_buf(buf: [u8; 8]) -> Self { + Self { data: buf } + } +} + +/// Represents timestamp 96 extension type. +/// This stores 64bit signed seconds and 32bit nanoseconds +pub struct Timestamp96 { + nanos: u32, + secs: i64, +} + +impl Timestamp96 { + pub fn new(seconds: i64, nanoseconds: u32) -> Self { + Self { + nanos: nanoseconds, + secs: seconds, + } + } + + pub fn nanos(&self) -> u32 { + self.nanos + } + + pub fn seconds(&self) -> i64 { + self.secs + } + + pub(crate) fn to_buf(&self) -> [u8; 12] { + let mut buf = [0u8; 12]; + buf[..4].copy_from_slice(&self.nanos.to_be_bytes()); + buf[4..].copy_from_slice(&self.secs.to_be_bytes()); + + buf + } + + pub(crate) fn from_buf(buf: [u8; 12]) -> Self { + let mut nano = [0u8; 4]; + nano.copy_from_slice(&buf[..4]); + + let mut second = [0u8; 8]; + second.copy_from_slice(&buf[4..]); + + Self { + nanos: u32::from_be_bytes(nano), + secs: i64::from_be_bytes(second), + } + } +} From 453daf461f6510e1d64bebb0eebfba9b2f500792 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Thu, 4 Sep 2025 22:24:55 +0000 Subject: [PATCH 09/57] test: add tests --- messagepack-core/src/decode/timestamp.rs | 152 +++++++++++++++++++++++ messagepack-core/src/encode/timestamp.rs | 51 ++++++++ messagepack-core/src/timestamp.rs | 40 +++++- 3 files changed, 239 insertions(+), 4 deletions(-) diff --git a/messagepack-core/src/decode/timestamp.rs b/messagepack-core/src/decode/timestamp.rs index 12db2e2..2f94091 100644 --- a/messagepack-core/src/decode/timestamp.rs +++ b/messagepack-core/src/decode/timestamp.rs @@ -86,3 +86,155 @@ impl<'a> Decode<'a> for Timestamp96 { Ok((timestamp, rest)) } } + +#[cfg(test)] +mod tests { + use super::*; + const TIMESTAMP_EXT_TYPE: u8 = 255; // -1 + + #[test] + fn decode_success_timestamp32() { + let secs: u32 = 1234567890; + let mut buf = vec![0xd6, TIMESTAMP_EXT_TYPE]; + buf.extend_from_slice(&secs.to_be_bytes()); + + let (ts, rest) = Timestamp32::decode(&buf).unwrap(); + assert_eq!(ts.seconds(), secs); + assert!(rest.is_empty()); + } + + #[test] + fn decode_failed_timestamp32_invalid_ext_type() { + let secs: u32 = 1; + let mut buf = vec![0xd6, 0]; // ext type != -1 + buf.extend_from_slice(&secs.to_be_bytes()); + + let err = Timestamp32::decode(&buf).unwrap_err(); + assert_eq!(err, Error::InvalidData); + } + + #[test] + fn decode_failed_timestamp32_eof_data() { + let secs: u32 = 123; + let mut buf = vec![0xd6, TIMESTAMP_EXT_TYPE]; + buf.extend_from_slice(&secs.to_be_bytes()[..3]); // 1 byte short + + let err = Timestamp32::decode(&buf).unwrap_err(); + assert_eq!(err, Error::EofData); + } + + #[test] + fn decode_success_timestamp64() { + let secs: u64 = 1234567890; + let nanos: u32 = 789; + + let data = ((nanos as u64) << 34) | secs; + let mut buf = vec![0xd7, TIMESTAMP_EXT_TYPE]; + buf.extend_from_slice(&data.to_be_bytes()); + + let (ts, rest) = Timestamp64::decode(&buf).unwrap(); + assert_eq!(ts.seconds(), secs); + assert_eq!(ts.nanos(), nanos); + assert!(rest.is_empty()); + } + + #[test] + fn decode_failed_timestamp64_unexpected_format() { + let mut buf = vec![0xd6, TIMESTAMP_EXT_TYPE]; // FixExt4, not FixExt8 + buf.extend_from_slice(&0u64.to_be_bytes()); + + let err = Timestamp64::decode(&buf).unwrap_err(); + assert_eq!(err, Error::UnexpectedFormat); + } + + #[test] + fn decode_failed_timestamp64_invalid_ext_type() { + let mut buf = vec![0xd7, 0]; // ext type != -1 + buf.extend_from_slice(&0u64.to_be_bytes()); + + let err = Timestamp64::decode(&buf).unwrap_err(); + assert_eq!(err, Error::InvalidData); + } + + #[test] + fn decode_failed_timestamp64_eof_data() { + let mut buf = vec![0xd7, TIMESTAMP_EXT_TYPE]; + buf.extend_from_slice(&[0u8; 7]); // 1 byte short + + let err = Timestamp64::decode(&buf).unwrap_err(); + assert_eq!(err, Error::EofData); + } + + #[test] + fn decode_success_timestamp96_positive() { + let secs: i64 = 123456; + let nanos: u32 = 789; + + let mut buf = vec![0xc7, 12, TIMESTAMP_EXT_TYPE]; + buf.extend_from_slice(&nanos.to_be_bytes()); + buf.extend_from_slice(&secs.to_be_bytes()); + + let (ts, rest) = Timestamp96::decode(&buf).unwrap(); + assert_eq!(ts.seconds(), secs); + assert_eq!(ts.nanos(), nanos); + assert!(rest.is_empty()); + } + + #[test] + fn decode_success_timestamp96_negative() { + let secs: i64 = -123; + let nanos: u32 = 42; + + let mut buf = vec![0xc7, 12, TIMESTAMP_EXT_TYPE]; + buf.extend_from_slice(&nanos.to_be_bytes()); + buf.extend_from_slice(&secs.to_be_bytes()); + + let (ts, rest) = Timestamp96::decode(&buf).unwrap(); + assert_eq!(ts.seconds(), secs); + assert_eq!(ts.nanos(), nanos); + assert!(rest.is_empty()); + } + + #[test] + fn decode_failed_timestamp96_unexpected_format() { + // FixExt8 header instead of Ext8 + let mut buf = vec![0xd7, TIMESTAMP_EXT_TYPE]; + buf.extend_from_slice(&[0u8; 8]); + + let err = Timestamp96::decode(&buf).unwrap_err(); + assert_eq!(err, Error::UnexpectedFormat); + } + + #[test] + fn decode_failed_timestamp96_invalid_length() { + // Ext8 length != 12 + let mut buf = vec![0xc7, 11, TIMESTAMP_EXT_TYPE]; + buf.extend_from_slice(&[0u8; 11]); + + let err = Timestamp96::decode(&buf).unwrap_err(); + assert_eq!(err, Error::InvalidData); + } + + #[test] + fn decode_failed_timestamp96_invalid_ext_type() { + let secs: i64 = 1; + let nanos: u32 = 2; + + let mut buf = vec![0xc7, 12, 0]; // ext type != -1 + buf.extend_from_slice(&nanos.to_be_bytes()); + buf.extend_from_slice(&secs.to_be_bytes()); + + let err = Timestamp96::decode(&buf).unwrap_err(); + assert_eq!(err, Error::InvalidData); + } + + #[test] + fn decode_failed_timestamp96_eof_data() { + // length says 12 but provide 11 + let mut buf = vec![0xc7, 12, TIMESTAMP_EXT_TYPE]; + buf.extend_from_slice(&[0u8; 11]); + + let err = Timestamp96::decode(&buf).unwrap_err(); + assert_eq!(err, Error::EofData); + } +} diff --git a/messagepack-core/src/encode/timestamp.rs b/messagepack-core/src/encode/timestamp.rs index 6cce462..853eb67 100644 --- a/messagepack-core/src/encode/timestamp.rs +++ b/messagepack-core/src/encode/timestamp.rs @@ -24,3 +24,54 @@ impl Encode for Timestamp96 { ExtensionEncoder::new(TIMESTAMP_EXTENSION_TYPE, &buf).encode(writer) } } + +#[cfg(test)] +mod tests { + use super::*; + + const TIMESTAMP_EXT_TYPE: u8 = 255; // -1 + + #[test] + fn encode_timestamp32() { + let ts = Timestamp32::new(123456); + let mut buf = vec![]; + + let n = ts.encode(&mut buf).unwrap(); + + let mut expected = vec![0xd6, TIMESTAMP_EXT_TYPE]; + expected.extend_from_slice(&123456_u32.to_be_bytes()); + + assert_eq!(buf, expected); + assert_eq!(n, expected.len()); + } + + #[test] + fn encode_timestamp64() { + let ts = Timestamp64::new(123456, 789).unwrap(); + let mut buf = vec![]; + + let n = ts.encode(&mut buf).unwrap(); + + let mut expected = vec![0xd7, TIMESTAMP_EXT_TYPE]; + let data = (789u64 << 34) | 123456; + expected.extend_from_slice(&data.to_be_bytes()); + + assert_eq!(buf, expected); + assert_eq!(n, expected.len()); + } + + #[test] + fn encode_timestamp96() { + let ts = Timestamp96::new(123456, 789); + let mut buf = vec![]; + + let n = ts.encode(&mut buf).unwrap(); + + let mut expected = vec![0xc7, 12, TIMESTAMP_EXT_TYPE]; + expected.extend_from_slice(&789_u32.to_be_bytes()); + expected.extend_from_slice(&123456_u64.to_be_bytes()); + + assert_eq!(buf, expected); + assert_eq!(n, expected.len()); + } +} diff --git a/messagepack-core/src/timestamp.rs b/messagepack-core/src/timestamp.rs index bc90a8b..86021e7 100644 --- a/messagepack-core/src/timestamp.rs +++ b/messagepack-core/src/timestamp.rs @@ -2,6 +2,7 @@ pub(crate) const TIMESTAMP_EXTENSION_TYPE: i8 = -1; /// Represents timestamp 32 extension type. /// This stores 32bit unsigned seconds +#[derive(Clone, Copy, Debug)] pub struct Timestamp32 { secs: u32, } @@ -15,7 +16,7 @@ impl Timestamp32 { self.secs } - pub(crate) fn to_buf(&self) -> [u8; 4] { + pub(crate) fn to_buf(self) -> [u8; 4] { self.secs.to_be_bytes() } @@ -28,11 +29,41 @@ impl Timestamp32 { /// Represents timestamp 64 extension type. /// This stores 34bit unsigned seconds and 30bit nanoseconds +#[derive(Clone, Copy, Debug)] pub struct Timestamp64 { data: [u8; 8], } +/// `seconds` or `nanos` cannot be represented +#[derive(Clone, Debug)] +pub struct Timestamp64Error { + pub seconds: u64, + pub nanos: u32, +} + impl Timestamp64 { + pub fn new(seconds: u64, nanos: u32) -> Result { + const SECONDS_MAX_LIMIT: u64 = 1 << 34; + + if seconds >= SECONDS_MAX_LIMIT { + return Err(Timestamp64Error { seconds, nanos }); + } + + const NANOS_MAX_LIMIT: u32 = 1 << 30; + if nanos >= NANOS_MAX_LIMIT { + return Err(Timestamp64Error { seconds, nanos }); + } + + let mut buf = [0u8; 8]; + buf[..].copy_from_slice(&seconds.to_be_bytes()); + + let nano = (nanos << 2).to_be_bytes(); + buf[..3].copy_from_slice(&nano[..3]); + buf[3] |= nano[3]; + + Ok(Self::from_buf(buf)) + } + pub fn nanos(&self) -> u32 { let mut buf = [0u8; 4]; buf.copy_from_slice(&self.data[..4]); @@ -42,7 +73,7 @@ impl Timestamp64 { pub fn seconds(&self) -> u64 { // 34bit mask - const MASK: u64 = 0x3FFFF; + const MASK: u64 = (1 << 34) - 1; let mut buf = [0u8; 8]; buf.copy_from_slice(&self.data[..]); let seconds = u64::from_be_bytes(buf); @@ -50,7 +81,7 @@ impl Timestamp64 { seconds & MASK } - pub(crate) fn to_buf(&self) -> [u8; 8] { + pub(crate) fn to_buf(self) -> [u8; 8] { self.data } @@ -61,6 +92,7 @@ impl Timestamp64 { /// Represents timestamp 96 extension type. /// This stores 64bit signed seconds and 32bit nanoseconds +#[derive(Clone, Copy, Debug)] pub struct Timestamp96 { nanos: u32, secs: i64, @@ -82,7 +114,7 @@ impl Timestamp96 { self.secs } - pub(crate) fn to_buf(&self) -> [u8; 12] { + pub(crate) fn to_buf(self) -> [u8; 12] { let mut buf = [0u8; 12]; buf[..4].copy_from_slice(&self.nanos.to_be_bytes()); buf[4..].copy_from_slice(&self.secs.to_be_bytes()); From c23fb876873e02259c453ecf35fc9f548b555c27 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Fri, 5 Sep 2025 12:12:17 +0000 Subject: [PATCH 10/57] fix: clippy error --- messagepack-serde/src/de/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/messagepack-serde/src/de/mod.rs b/messagepack-serde/src/de/mod.rs index 7c9e278..4a268fe 100644 --- a/messagepack-serde/src/de/mod.rs +++ b/messagepack-serde/src/de/mod.rs @@ -113,7 +113,7 @@ where reader.read_to_end(&mut buf)?; let mut deserializer = Deserializer::from_slice(&buf); - T::deserialize(&mut deserializer).map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e)) + T::deserialize(&mut deserializer).map_err(std::io::Error::other) } impl<'de> de::Deserializer<'de> for &mut Deserializer<'de> { From a1a6e6003514b963d6c0a066aa8ce3a543a79d56 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Fri, 5 Sep 2025 12:32:59 +0000 Subject: [PATCH 11/57] chore: ignore bench --- .github/codecov.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/codecov.yml b/.github/codecov.yml index 5706c46..4e280a3 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -6,6 +6,9 @@ comment: require_head: true hide_project_coverage: false # [true :: only show coverage on the git diff aka patch coverage] +ignore: + - "messagepack-bench" + coverage: status: project: From ff13009536bcb16aaae18d3da508d070340deaf2 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Fri, 5 Sep 2025 12:43:15 +0000 Subject: [PATCH 12/57] chore: rename --- messagepack-core/src/encode/array.rs | 6 ++--- messagepack-core/src/encode/bin.rs | 8 +++--- messagepack-core/src/encode/bool.rs | 4 +-- messagepack-core/src/encode/extension.rs | 32 ++++++++++++------------ messagepack-core/src/encode/float.rs | 8 +++--- messagepack-core/src/encode/int.rs | 16 ++++++------ messagepack-core/src/encode/map.rs | 6 ++--- messagepack-core/src/encode/mod.rs | 2 +- messagepack-core/src/encode/nil.rs | 2 +- messagepack-core/src/encode/str.rs | 16 ++++++------ messagepack-core/src/io.rs | 30 ++++------------------ messagepack-serde/src/value/extension.rs | 2 +- 12 files changed, 56 insertions(+), 76 deletions(-) diff --git a/messagepack-core/src/encode/array.rs b/messagepack-core/src/encode/array.rs index c313034..6d2ef70 100644 --- a/messagepack-core/src/encode/array.rs +++ b/messagepack-core/src/encode/array.rs @@ -14,18 +14,18 @@ impl Encode for ArrayFormatEncoder { match self.0 { 0x00..=0b1111 => { let cast = self.0 as u8; - writer.write_bytes(&[Format::FixArray(cast).as_byte()])?; + writer.write(&[Format::FixArray(cast).as_byte()])?; Ok(1) } 0x10..=0xffff => { let cast = (self.0 as u16).to_be_bytes(); - writer.write_bytes(&[Format::Array16.as_byte(), cast[0], cast[1]])?; + writer.write(&[Format::Array16.as_byte(), cast[0], cast[1]])?; Ok(3) } 0x10000..=0xffffffff => { let cast = (self.0 as u32).to_be_bytes(); - writer.write_bytes(&[ + writer.write(&[ Format::Array32.as_byte(), cast[0], cast[1], diff --git a/messagepack-core/src/encode/bin.rs b/messagepack-core/src/encode/bin.rs index 0de16a1..8283cfd 100644 --- a/messagepack-core/src/encode/bin.rs +++ b/messagepack-core/src/encode/bin.rs @@ -17,17 +17,17 @@ impl Encode for BinaryEncoder<'_> { let format_len = match self_len { 0x00..=0xff => { let cast = self_len as u8; - writer.write_bytes(&[Format::Bin8.as_byte(), cast])?; + writer.write(&[Format::Bin8.as_byte(), cast])?; Ok(2) } 0x100..=0xffff => { let cast = (self_len as u16).to_be_bytes(); - writer.write_bytes(&[Format::Bin16.as_byte(), cast[0], cast[1]])?; + writer.write(&[Format::Bin16.as_byte(), cast[0], cast[1]])?; Ok(3) } 0x10000..=0xffffffff => { let cast = (self_len as u32).to_be_bytes(); - writer.write_bytes(&[ + writer.write(&[ Format::Bin32.as_byte(), cast[0], cast[1], @@ -40,7 +40,7 @@ impl Encode for BinaryEncoder<'_> { _ => Err(Error::InvalidFormat), }?; - writer.write_bytes(self.0)?; + writer.write(self.0)?; Ok(format_len + self_len) } } diff --git a/messagepack-core/src/encode/bool.rs b/messagepack-core/src/encode/bool.rs index 3814a52..986d9c2 100644 --- a/messagepack-core/src/encode/bool.rs +++ b/messagepack-core/src/encode/bool.rs @@ -5,11 +5,11 @@ impl Encode for bool { fn encode(&self, writer: &mut W) -> Result::Error> { match self { true => { - writer.write_bytes(&Format::True.as_slice())?; + writer.write(&Format::True.as_slice())?; Ok(1) } false => { - writer.write_bytes(&Format::False.as_slice())?; + writer.write(&Format::False.as_slice())?; Ok(1) } } diff --git a/messagepack-core/src/encode/extension.rs b/messagepack-core/src/encode/extension.rs index 5ba0476..344bcf9 100644 --- a/messagepack-core/src/encode/extension.rs +++ b/messagepack-core/src/encode/extension.rs @@ -39,51 +39,51 @@ impl Encode for ExtensionEncoder<'_> { match data_len { 1 => { - writer.write_bytes(&[Format::FixExt1.as_byte(), type_byte])?; - writer.write_bytes(self.data)?; + writer.write(&[Format::FixExt1.as_byte(), type_byte])?; + writer.write(self.data)?; Ok(2 + data_len) } 2 => { - writer.write_bytes(&[Format::FixExt2.as_byte(), type_byte])?; - writer.write_bytes(self.data)?; + writer.write(&[Format::FixExt2.as_byte(), type_byte])?; + writer.write(self.data)?; Ok(2 + data_len) } 4 => { - writer.write_bytes(&[Format::FixExt4.as_byte(), type_byte])?; - writer.write_bytes(self.data)?; + writer.write(&[Format::FixExt4.as_byte(), type_byte])?; + writer.write(self.data)?; Ok(2 + data_len) } 8 => { - writer.write_bytes(&[Format::FixExt8.as_byte(), type_byte])?; - writer.write_bytes(self.data)?; + writer.write(&[Format::FixExt8.as_byte(), type_byte])?; + writer.write(self.data)?; Ok(2 + data_len) } 16 => { - writer.write_bytes(&[Format::FixExt16.as_byte(), type_byte])?; - writer.write_bytes(self.data)?; + writer.write(&[Format::FixExt16.as_byte(), type_byte])?; + writer.write(self.data)?; Ok(2 + data_len) } 0..=0xff => { let cast = data_len as u8; - writer.write_bytes(&[Format::Ext8.as_byte(), cast, type_byte])?; - writer.write_bytes(self.data)?; + writer.write(&[Format::Ext8.as_byte(), cast, type_byte])?; + writer.write(self.data)?; Ok(3 + data_len) } 0x100..=U16_MAX => { let cast = (data_len as u16).to_be_bytes(); - writer.write_bytes(&[Format::Ext16.as_byte(), cast[0], cast[1], type_byte])?; - writer.write_bytes(self.data)?; + writer.write(&[Format::Ext16.as_byte(), cast[0], cast[1], type_byte])?; + writer.write(self.data)?; Ok(4 + data_len) } 0x10000..=U32_MAX => { let cast = (data_len as u32).to_be_bytes(); - writer.write_bytes(&[ + writer.write(&[ Format::Ext32.as_byte(), cast[0], cast[1], @@ -91,7 +91,7 @@ impl Encode for ExtensionEncoder<'_> { cast[3], type_byte, ])?; - writer.write_bytes(self.data)?; + writer.write(self.data)?; Ok(6 + data_len) } diff --git a/messagepack-core/src/encode/float.rs b/messagepack-core/src/encode/float.rs index add5135..ad52ad3 100644 --- a/messagepack-core/src/encode/float.rs +++ b/messagepack-core/src/encode/float.rs @@ -6,8 +6,8 @@ where W: IoWrite, { fn encode(&self, writer: &mut W) -> Result::Error> { - writer.write_bytes(&Format::Float32.as_slice())?; - writer.write_bytes(&self.to_be_bytes())?; + writer.write(&Format::Float32.as_slice())?; + writer.write(&self.to_be_bytes())?; Ok(5) } } @@ -17,8 +17,8 @@ where W: IoWrite, { fn encode(&self, writer: &mut W) -> Result::Error> { - writer.write_bytes(&Format::Float64.as_slice())?; - writer.write_bytes(&self.to_be_bytes())?; + writer.write(&Format::Float64.as_slice())?; + writer.write(&self.to_be_bytes())?; Ok(9) } } diff --git a/messagepack-core/src/encode/int.rs b/messagepack-core/src/encode/int.rs index 1d1cffb..0ba52c9 100644 --- a/messagepack-core/src/encode/int.rs +++ b/messagepack-core/src/encode/int.rs @@ -10,13 +10,13 @@ where fn encode(&self, writer: &mut W) -> Result { match self { 0x00..=0x7f => { - writer.write_bytes(&Format::PositiveFixInt(*self).as_slice())?; + writer.write(&Format::PositiveFixInt(*self).as_slice())?; Ok(1) } _ => { - writer.write_bytes(&Format::Uint8.as_slice())?; - writer.write_bytes(&self.to_be_bytes())?; + writer.write(&Format::Uint8.as_slice())?; + writer.write(&self.to_be_bytes())?; Ok(2) } } @@ -42,12 +42,12 @@ where fn encode(&self, writer: &mut W) -> Result { match self { -32..=-1 => { - writer.write_bytes(&Format::NegativeFixInt(*self).as_slice())?; + writer.write(&Format::NegativeFixInt(*self).as_slice())?; Ok(1) } _ => { - writer.write_bytes(&Format::Int8.as_slice())?; - writer.write_bytes(&self.to_be_bytes())?; + writer.write(&Format::Int8.as_slice())?; + writer.write(&self.to_be_bytes())?; Ok(2) } @@ -62,8 +62,8 @@ macro_rules! impl_encode_int { W: IoWrite, { fn encode(&self, writer: &mut W) -> Result { - writer.write_bytes(&$format.as_slice())?; - writer.write_bytes(&self.to_be_bytes())?; + writer.write(&$format.as_slice())?; + writer.write(&self.to_be_bytes())?; Ok($size) } } diff --git a/messagepack-core/src/encode/map.rs b/messagepack-core/src/encode/map.rs index 7efb07d..2e3ffd4 100644 --- a/messagepack-core/src/encode/map.rs +++ b/messagepack-core/src/encode/map.rs @@ -37,19 +37,19 @@ impl Encode for MapFormatEncoder { match self.0 { 0x00..=0xf => { let cast = self.0 as u8; - writer.write_bytes(&[Format::FixMap(cast).as_byte()])?; + writer.write(&[Format::FixMap(cast).as_byte()])?; Ok(1) } 0x10..=0xffff => { let cast = (self.0 as u16).to_be_bytes(); - writer.write_bytes(&[Format::Map16.as_byte(), cast[0], cast[1]])?; + writer.write(&[Format::Map16.as_byte(), cast[0], cast[1]])?; Ok(3) } 0x10000..=0xffffffff => { let cast = (self.0 as u32).to_be_bytes(); - writer.write_bytes(&[ + writer.write(&[ Format::Map32.as_byte(), cast[0], cast[1], diff --git a/messagepack-core/src/encode/mod.rs b/messagepack-core/src/encode/mod.rs index 2221394..89b30ae 100644 --- a/messagepack-core/src/encode/mod.rs +++ b/messagepack-core/src/encode/mod.rs @@ -79,7 +79,7 @@ where W: IoWrite, { fn encode(&self, writer: &mut W) -> Result::Error> { - writer.write_bytes(&self.as_slice())?; + writer.write(&self.as_slice())?; Ok(1) } } diff --git a/messagepack-core/src/encode/nil.rs b/messagepack-core/src/encode/nil.rs index c88dd6e..a3c2004 100644 --- a/messagepack-core/src/encode/nil.rs +++ b/messagepack-core/src/encode/nil.rs @@ -5,7 +5,7 @@ pub struct NilEncoder; impl Encode for NilEncoder { fn encode(&self, writer: &mut W) -> Result::Error> { - writer.write_bytes(&Format::Nil.as_slice())?; + writer.write(&Format::Nil.as_slice())?; Ok(1) } } diff --git a/messagepack-core/src/encode/str.rs b/messagepack-core/src/encode/str.rs index 12e7e76..d7bdb66 100644 --- a/messagepack-core/src/encode/str.rs +++ b/messagepack-core/src/encode/str.rs @@ -9,25 +9,25 @@ impl Encode for StrFormatEncoder { match self.0 { 0x00..=31 => { let cast = self.0 as u8; - writer.write_bytes(&Format::FixStr(cast).as_slice())?; + writer.write(&Format::FixStr(cast).as_slice())?; Ok(1) } 32..=0xff => { let cast = self.0 as u8; - writer.write_bytes(&Format::Str8.as_slice())?; - writer.write_bytes(&cast.to_be_bytes())?; + writer.write(&Format::Str8.as_slice())?; + writer.write(&cast.to_be_bytes())?; Ok(2) } 0x100..=0xffff => { let cast = self.0 as u16; - writer.write_bytes(&Format::Str16.as_slice())?; - writer.write_bytes(&cast.to_be_bytes())?; + writer.write(&Format::Str16.as_slice())?; + writer.write(&cast.to_be_bytes())?; Ok(3) } 0x10000..=0xffffffff => { let cast = self.0 as u32; - writer.write_bytes(&Format::Str32.as_slice())?; - writer.write_bytes(&cast.to_be_bytes())?; + writer.write(&Format::Str32.as_slice())?; + writer.write(&cast.to_be_bytes())?; Ok(5) } _ => Err(Error::InvalidFormat), @@ -39,7 +39,7 @@ pub struct StrDataEncoder<'a>(pub &'a str); impl Encode for StrDataEncoder<'_> { fn encode(&self, writer: &mut W) -> Result::Error> { let data = self.0.as_bytes(); - writer.write_bytes(data)?; + writer.write(data)?; Ok(self.0.len()) } } diff --git a/messagepack-core/src/io.rs b/messagepack-core/src/io.rs index 95c5d69..99be755 100644 --- a/messagepack-core/src/io.rs +++ b/messagepack-core/src/io.rs @@ -1,13 +1,6 @@ pub trait IoWrite { type Error: core::error::Error; - fn write_byte(&mut self, byte: u8) -> Result<(), Self::Error>; - fn write_bytes(&mut self, buf: &[u8]) -> Result<(), Self::Error> { - for byte in buf { - self.write_byte(*byte)?; - } - - Ok(()) - } + fn write(&mut self, buf: &[u8]) -> Result<(), Self::Error>; } /// `SliceWriter` Error @@ -45,17 +38,7 @@ impl<'a> SliceWriter<'a> { impl IoWrite for SliceWriter<'_> { type Error = WError; - fn write_byte(&mut self, byte: u8) -> Result<(), Self::Error> { - if self.len() >= 1 { - self.buf[self.cursor] = byte; - self.cursor += 1; - Ok(()) - } else { - Err(WError::BufferFull) - } - } - - fn write_bytes(&mut self, buf: &[u8]) -> Result<(), Self::Error> { + fn write(&mut self, buf: &[u8]) -> Result<(), Self::Error> { if self.len() >= buf.len() { let to = &mut self.buf[self.cursor..self.cursor + buf.len()]; to.copy_from_slice(buf); @@ -74,11 +57,8 @@ where { type Error = std::io::Error; - fn write_byte(&mut self, byte: u8) -> Result<(), Self::Error> { - match self.write_all(&[byte]) { - Ok(_) => Ok(()), - Err(e) => Err(e), - } + fn write(&mut self, buf: &[u8]) -> Result<(), Self::Error> { + self.write_all(buf) } } @@ -91,6 +71,6 @@ mod tests { fn buffer_full() { let buf: &mut [u8] = &mut [0u8]; let mut writer = SliceWriter::from_slice(buf); - writer.write_bytes(&[1, 2]).unwrap(); + writer.write(&[1, 2]).unwrap(); } } diff --git a/messagepack-serde/src/value/extension.rs b/messagepack-serde/src/value/extension.rs index 8cb4491..15c304b 100644 --- a/messagepack-serde/src/value/extension.rs +++ b/messagepack-serde/src/value/extension.rs @@ -121,7 +121,7 @@ where } fn serialize_bytes(self, v: &[u8]) -> Result { - self.writer.write_bytes(v).map_err(CoreError::Io)?; + self.writer.write(v).map_err(CoreError::Io)?; *self.length += v.len(); Ok(()) } From b309928373a6d1f856bd6b824d0afc93a51845c9 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Fri, 5 Sep 2025 13:11:50 +0000 Subject: [PATCH 13/57] refactor: use slice --- messagepack-core/src/encode/array.rs | 61 +++------------------------- messagepack-core/src/encode/mod.rs | 1 - messagepack-serde/src/ser/mod.rs | 6 +-- 3 files changed, 9 insertions(+), 59 deletions(-) diff --git a/messagepack-core/src/encode/array.rs b/messagepack-core/src/encode/array.rs index 6d2ef70..5b3dc9e 100644 --- a/messagepack-core/src/encode/array.rs +++ b/messagepack-core/src/encode/array.rs @@ -1,14 +1,8 @@ -use core::{cell::RefCell, marker::PhantomData}; - use super::{Encode, Error, Result}; use crate::{formats::Format, io::IoWrite}; pub struct ArrayFormatEncoder(pub usize); -impl ArrayFormatEncoder { - pub fn new(size: usize) -> Self { - Self(size) - } -} + impl Encode for ArrayFormatEncoder { fn encode(&self, writer: &mut W) -> Result::Error> { match self.0 { @@ -40,57 +34,17 @@ impl Encode for ArrayFormatEncoder { } } -pub struct ArrayDataEncoder { - data: RefCell, - _phantom: PhantomData<(I, V)>, -} - -impl ArrayDataEncoder { - pub fn new(data: I) -> Self { - ArrayDataEncoder { - data: RefCell::new(data), - _phantom: Default::default(), - } - } -} - -impl Encode for ArrayDataEncoder +impl Encode for &[V] where W: IoWrite, - I: Iterator, V: Encode, { fn encode(&self, writer: &mut W) -> Result::Error> { + let format_len = ArrayFormatEncoder(self.len()).encode(writer)?; let array_len = self - .data - .borrow_mut() - .by_ref() + .iter() .map(|v| v.encode(writer)) .try_fold(0, |acc, v| v.map(|n| acc + n))?; - Ok(array_len) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub struct ArrayEncoder<'array, V>(&'array [V]); - -impl<'array, V> core::ops::Deref for ArrayEncoder<'array, V> { - type Target = &'array [V]; - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl Encode for ArrayEncoder<'_, V> -where - W: IoWrite, - V: Encode, -{ - fn encode(&self, writer: &mut W) -> Result::Error> { - let self_len = self.len(); - let format_len = ArrayFormatEncoder(self_len).encode(writer)?; - - let array_len = ArrayDataEncoder::new(self.iter()).encode(writer)?; Ok(format_len + array_len) } } @@ -107,10 +61,9 @@ mod tests { #[case] expected: E, ) { let expected = expected.as_ref(); - let encoder = ArrayEncoder(value.as_ref()); let mut buf = vec![]; - let n = encoder.encode(&mut buf).unwrap(); + let n = value.as_ref().encode(&mut buf).unwrap(); assert_eq!(buf, expected); assert_eq!(n, expected.len()); } @@ -131,10 +84,8 @@ mod tests { .cloned() .collect::>(); - let encoder = ArrayEncoder(data.as_ref()); - let mut buf = vec![]; - let n = encoder.encode(&mut buf).unwrap(); + let n = data.as_ref().encode(&mut buf).unwrap(); assert_eq!(&buf, &expected); assert_eq!(n, expected.len()); diff --git a/messagepack-core/src/encode/mod.rs b/messagepack-core/src/encode/mod.rs index 89b30ae..2e972a4 100644 --- a/messagepack-core/src/encode/mod.rs +++ b/messagepack-core/src/encode/mod.rs @@ -9,7 +9,6 @@ pub mod nil; pub mod str; mod timestamp; -pub use array::{ArrayDataEncoder, ArrayEncoder, ArrayFormatEncoder}; pub use bin::BinaryEncoder; pub use extension::ExtensionEncoder; pub use map::{MapDataEncoder, MapEncoder, MapFormatEncoder, MapSliceEncoder}; diff --git a/messagepack-serde/src/ser/mod.rs b/messagepack-serde/src/ser/mod.rs index c2fe95b..e0e1323 100644 --- a/messagepack-serde/src/ser/mod.rs +++ b/messagepack-serde/src/ser/mod.rs @@ -10,7 +10,7 @@ use crate::value::extension::{EXTENSION_STRUCT_NAME, SerializeExt}; pub use error::{CoreError, Error}; use messagepack_core::{ Encode, SliceWriter, - encode::{ArrayFormatEncoder, BinaryEncoder, MapFormatEncoder, NilEncoder}, + encode::{BinaryEncoder, MapFormatEncoder, NilEncoder, array::ArrayFormatEncoder}, io::{IoWrite, WError}, }; @@ -274,7 +274,7 @@ where fn serialize_seq(self, len: Option) -> Result { let len = len.ok_or(Error::SeqLenNone)?; - self.current_length += ArrayFormatEncoder::new(len).encode(self.writer)?; + self.current_length += ArrayFormatEncoder(len).encode(self.writer)?; Ok(seq::SerializeSeq::new(self)) } @@ -299,7 +299,7 @@ where ) -> Result { self.current_length += MapFormatEncoder::new(1).encode(self.writer)?; self.serialize_str(variant)?; - self.current_length += ArrayFormatEncoder::new(len).encode(self.writer)?; + self.current_length += ArrayFormatEncoder(len).encode(self.writer)?; Ok(seq::SerializeSeq::new(self)) } From 4b53629ab064437146bb55cebc1c87768e63a035 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Fri, 5 Sep 2025 13:27:14 +0000 Subject: [PATCH 14/57] fix --- messagepack-serde/src/value/value_.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/messagepack-serde/src/value/value_.rs b/messagepack-serde/src/value/value_.rs index 9a90196..be067df 100644 --- a/messagepack-serde/src/value/value_.rs +++ b/messagepack-serde/src/value/value_.rs @@ -164,8 +164,9 @@ impl<'de> serde::Deserialize<'de> for ValueRef<'de> { A: serde::de::SeqAccess<'de>, { let mut buf = Vec::new(); + - while let Ok(Some(v)) = seq.next_element::() { + while let Some(v) = seq.next_element::()? { buf.push(v); } @@ -178,7 +179,7 @@ impl<'de> serde::Deserialize<'de> for ValueRef<'de> { { let mut buf = Vec::new(); - while let Ok(Some(v)) = map.next_entry() { + while let Some(v) = map.next_entry()? { buf.push(v); } From 45f06a9970a8c9fcbebbd49d36a5dad45623d137 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Sat, 6 Sep 2025 04:54:18 +0000 Subject: [PATCH 15/57] chore: move to extension --- messagepack-core/src/decode/extension.rs | 72 ----- messagepack-core/src/decode/mod.rs | 2 - messagepack-core/src/encode/extension.rs | 159 ------------ messagepack-core/src/encode/mod.rs | 2 - messagepack-core/src/encode/timestamp.rs | 9 +- messagepack-core/src/extension.rs | 317 +++++++++++++++++++++++ messagepack-core/src/lib.rs | 1 + messagepack-serde/src/value/extension.rs | 6 +- 8 files changed, 326 insertions(+), 242 deletions(-) delete mode 100644 messagepack-core/src/decode/extension.rs delete mode 100644 messagepack-core/src/encode/extension.rs create mode 100644 messagepack-core/src/extension.rs diff --git a/messagepack-core/src/decode/extension.rs b/messagepack-core/src/decode/extension.rs deleted file mode 100644 index 0eb1148..0000000 --- a/messagepack-core/src/decode/extension.rs +++ /dev/null @@ -1,72 +0,0 @@ -use super::{Decode, Error, NbyteReader, Result}; -use crate::formats::Format; - -pub struct Extension<'a> { - pub r#type: i8, - pub data: &'a [u8], -} - -pub struct ExtensionDecoder; - -impl<'a> Decode<'a> for ExtensionDecoder { - type Value = Extension<'a>; - fn decode(buf: &'a [u8]) -> Result<(Self::Value, &'a [u8])> { - let (format, buf) = Format::decode(buf)?; - match format { - Format::FixExt1 - | Format::FixExt2 - | Format::FixExt4 - | Format::FixExt8 - | Format::FixExt16 - | Format::Ext8 - | Format::Ext16 - | Format::Ext32 => Self::decode_with_format(format, buf), - _ => Err(Error::UnexpectedFormat), - } - } - fn decode_with_format(format: Format, buf: &'a [u8]) -> Result<(Self::Value, &'a [u8])> { - let (ext_type, buf) = buf.split_first().ok_or(Error::EofData)?; - let (len, buf) = match format { - Format::FixExt1 => (1, buf), - Format::FixExt2 => (2, buf), - Format::FixExt4 => (4, buf), - Format::FixExt8 => (8, buf), - Format::FixExt16 => (16, buf), - Format::Ext8 => NbyteReader::<1>::read(buf)?, - Format::Ext16 => NbyteReader::<2>::read(buf)?, - Format::Ext32 => NbyteReader::<4>::read(buf)?, - _ => return Err(Error::UnexpectedFormat), - }; - let (data, rest) = buf.split_at_checked(len).ok_or(Error::EofData)?; - let ext = Extension { - r#type: (*ext_type) as i8, - data, - }; - Ok((ext, rest)) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - const TIMESTAMP32: &[u8] = &[ - 0xd6, // FixExt4 - 0xff, // Timestamp type = -1 - // Feb 22 2022 22:22:22 GMT+0000 - // Unix sec = 1645568542 - 0x62, 0x15, 0x62, 0x1e, - ]; - - #[test] - fn decode_fix_ext4() { - let (ext, rest) = ExtensionDecoder::decode(TIMESTAMP32).unwrap(); - - let expect_type = -1; - let expect_data = 1645568542; - assert_eq!(ext.r#type, expect_type); - let data_u32 = u32::from_be_bytes(ext.data.try_into().unwrap()); - assert_eq!(data_u32, expect_data); - assert_eq!(rest.len(), 0); - } -} diff --git a/messagepack-core/src/decode/mod.rs b/messagepack-core/src/decode/mod.rs index a02660f..d6ddb25 100644 --- a/messagepack-core/src/decode/mod.rs +++ b/messagepack-core/src/decode/mod.rs @@ -5,8 +5,6 @@ pub use array::ArrayDecoder; mod bin; pub use bin::BinDecoder; mod bool; -mod extension; -pub use extension::{Extension, ExtensionDecoder}; mod float; mod int; mod map; diff --git a/messagepack-core/src/encode/extension.rs b/messagepack-core/src/encode/extension.rs deleted file mode 100644 index 344bcf9..0000000 --- a/messagepack-core/src/encode/extension.rs +++ /dev/null @@ -1,159 +0,0 @@ -use super::{Encode, Error, Result}; -use crate::{formats::Format, io::IoWrite}; - -pub const U8_MAX: usize = u8::MAX as usize; -pub const U16_MAX: usize = u16::MAX as usize; -pub const U32_MAX: usize = u32::MAX as usize; - -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub struct ExtensionEncoder<'data> { - r#type: i8, - data: &'data [u8], -} - -impl<'data> ExtensionEncoder<'data> { - pub fn new(r#type: i8, data: &'data [u8]) -> Self { - Self { r#type, data } - } - - pub fn to_format(&self) -> Result { - let format = match self.data.len() { - 1 => Format::FixExt1, - 2 => Format::FixExt2, - 4 => Format::FixExt4, - 8 => Format::FixExt8, - 16 => Format::FixExt16, - 0..U8_MAX => Format::Ext8, - U8_MAX..U16_MAX => Format::Ext16, - U16_MAX..U32_MAX => Format::Ext32, - _ => return Err(Error::InvalidFormat), - }; - Ok(format) - } -} - -impl Encode for ExtensionEncoder<'_> { - fn encode(&self, writer: &mut W) -> Result { - let data_len = self.data.len(); - let type_byte = self.r#type.to_be_bytes()[0]; - - match data_len { - 1 => { - writer.write(&[Format::FixExt1.as_byte(), type_byte])?; - writer.write(self.data)?; - - Ok(2 + data_len) - } - 2 => { - writer.write(&[Format::FixExt2.as_byte(), type_byte])?; - writer.write(self.data)?; - - Ok(2 + data_len) - } - 4 => { - writer.write(&[Format::FixExt4.as_byte(), type_byte])?; - writer.write(self.data)?; - Ok(2 + data_len) - } - 8 => { - writer.write(&[Format::FixExt8.as_byte(), type_byte])?; - writer.write(self.data)?; - - Ok(2 + data_len) - } - 16 => { - writer.write(&[Format::FixExt16.as_byte(), type_byte])?; - writer.write(self.data)?; - - Ok(2 + data_len) - } - 0..=0xff => { - let cast = data_len as u8; - writer.write(&[Format::Ext8.as_byte(), cast, type_byte])?; - writer.write(self.data)?; - - Ok(3 + data_len) - } - 0x100..=U16_MAX => { - let cast = (data_len as u16).to_be_bytes(); - writer.write(&[Format::Ext16.as_byte(), cast[0], cast[1], type_byte])?; - writer.write(self.data)?; - - Ok(4 + data_len) - } - 0x10000..=U32_MAX => { - let cast = (data_len as u32).to_be_bytes(); - writer.write(&[ - Format::Ext32.as_byte(), - cast[0], - cast[1], - cast[2], - cast[3], - type_byte, - ])?; - writer.write(self.data)?; - - Ok(6 + data_len) - } - _ => Err(Error::InvalidFormat), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use rstest::rstest; - - #[rstest] - #[case(0xd4,123,[0x12])] - #[case(0xd5,123,[0x12,0x34])] - #[case(0xd6,123,[0x12,0x34,0x56,0x78])] - #[case(0xd7,123,[0x12;8])] - #[case(0xd8,123,[0x12;16])] - fn encode_ext_fixed>(#[case] marker: u8, #[case] ty: i8, #[case] data: D) { - let expected = marker - .to_be_bytes() - .iter() - .chain(ty.to_be_bytes().iter()) - .chain(data.as_ref()) - .cloned() - .collect::>(); - - let encoder = ExtensionEncoder::new(ty, data.as_ref()); - - let mut buf = vec![]; - let n = encoder.encode(&mut buf).unwrap(); - - assert_eq!(&buf, &expected); - assert_eq!(n, expected.len()); - } - - #[rstest] - #[case(0xc7_u8.to_be_bytes(),123,5u8.to_be_bytes(),[0x12;5])] - #[case(0xc8_u8.to_be_bytes(),123,65535_u16.to_be_bytes(),[0x34;65535])] - #[case(0xc9_u8.to_be_bytes(),123,65536_u32.to_be_bytes(),[0x56;65536])] - fn encode_ext_sized, S: AsRef<[u8]>, D: AsRef<[u8]>>( - #[case] marker: M, - #[case] ty: i8, - #[case] size: S, - #[case] data: D, - ) { - let expected = marker - .as_ref() - .iter() - .chain(size.as_ref()) - .chain(ty.to_be_bytes().iter()) - .chain(data.as_ref()) - .cloned() - .collect::>(); - - let encoder = ExtensionEncoder::new(ty, data.as_ref()); - - let mut buf = vec![]; - let n = encoder.encode(&mut buf).unwrap(); - - assert_eq!(&buf, &expected); - assert_eq!(n, expected.len()); - } -} diff --git a/messagepack-core/src/encode/mod.rs b/messagepack-core/src/encode/mod.rs index 2e972a4..f3c09ef 100644 --- a/messagepack-core/src/encode/mod.rs +++ b/messagepack-core/src/encode/mod.rs @@ -1,7 +1,6 @@ pub mod array; pub mod bin; pub mod bool; -pub mod extension; pub mod float; pub mod int; pub mod map; @@ -10,7 +9,6 @@ pub mod str; mod timestamp; pub use bin::BinaryEncoder; -pub use extension::ExtensionEncoder; pub use map::{MapDataEncoder, MapEncoder, MapFormatEncoder, MapSliceEncoder}; pub use nil::NilEncoder; diff --git a/messagepack-core/src/encode/timestamp.rs b/messagepack-core/src/encode/timestamp.rs index 853eb67..77f6bb3 100644 --- a/messagepack-core/src/encode/timestamp.rs +++ b/messagepack-core/src/encode/timestamp.rs @@ -1,5 +1,6 @@ -use super::{Encode, Result, extension::ExtensionEncoder}; +use super::{Encode, Result}; use crate::{ + extension::FixedExtension, io::IoWrite, timestamp::{TIMESTAMP_EXTENSION_TYPE, Timestamp32, Timestamp64, Timestamp96}, }; @@ -7,21 +8,21 @@ use crate::{ impl Encode for Timestamp32 { fn encode(&self, writer: &mut W) -> Result { let buf = self.to_buf(); - ExtensionEncoder::new(TIMESTAMP_EXTENSION_TYPE, &buf).encode(writer) + FixedExtension::new_fixed(TIMESTAMP_EXTENSION_TYPE, buf).encode(writer) } } impl Encode for Timestamp64 { fn encode(&self, writer: &mut W) -> Result { let buf = self.to_buf(); - ExtensionEncoder::new(TIMESTAMP_EXTENSION_TYPE, &buf).encode(writer) + FixedExtension::new_fixed(TIMESTAMP_EXTENSION_TYPE, buf).encode(writer) } } impl Encode for Timestamp96 { fn encode(&self, writer: &mut W) -> Result { let buf = self.to_buf(); - ExtensionEncoder::new(TIMESTAMP_EXTENSION_TYPE, &buf).encode(writer) + FixedExtension::new_fixed(TIMESTAMP_EXTENSION_TYPE, buf).encode(writer) } } diff --git a/messagepack-core/src/extension.rs b/messagepack-core/src/extension.rs new file mode 100644 index 0000000..941c08b --- /dev/null +++ b/messagepack-core/src/extension.rs @@ -0,0 +1,317 @@ +use crate::decode::{self, NbyteReader}; +use crate::encode; +use crate::{Decode, Encode, formats::Format, io::IoWrite}; + +pub const U8_MAX: usize = u8::MAX as usize; +pub const U16_MAX: usize = u16::MAX as usize; +pub const U32_MAX: usize = u32::MAX as usize; + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct ExtensionRef<'a> { + pub r#type: i8, + pub data: &'a [u8], +} + +impl<'a> ExtensionRef<'a> { + pub fn new(r#type: i8, data: &'a [u8]) -> Self { + Self { r#type, data } + } + + pub fn to_format(&self) -> core::result::Result> { + let format = match self.data.len() { + 1 => Format::FixExt1, + 2 => Format::FixExt2, + 4 => Format::FixExt4, + 8 => Format::FixExt8, + 16 => Format::FixExt16, + 0..U8_MAX => Format::Ext8, + U8_MAX..U16_MAX => Format::Ext16, + U16_MAX..U32_MAX => Format::Ext32, + _ => return Err(encode::Error::InvalidFormat), + }; + Ok(format) + } +} + +impl<'a, W: IoWrite> Encode for ExtensionRef<'a> { + fn encode(&self, writer: &mut W) -> core::result::Result> { + let data_len = self.data.len(); + let type_byte = self.r#type.to_be_bytes()[0]; + + match data_len { + 1 => { + writer.write(&[Format::FixExt1.as_byte(), type_byte])?; + writer.write(self.data)?; + Ok(2 + data_len) + } + 2 => { + writer.write(&[Format::FixExt2.as_byte(), type_byte])?; + writer.write(self.data)?; + Ok(2 + data_len) + } + 4 => { + writer.write(&[Format::FixExt4.as_byte(), type_byte])?; + writer.write(self.data)?; + Ok(2 + data_len) + } + 8 => { + writer.write(&[Format::FixExt8.as_byte(), type_byte])?; + writer.write(self.data)?; + Ok(2 + data_len) + } + 16 => { + writer.write(&[Format::FixExt16.as_byte(), type_byte])?; + writer.write(self.data)?; + Ok(2 + data_len) + } + 0..=0xff => { + let cast = data_len as u8; + writer.write(&[Format::Ext8.as_byte(), cast, type_byte])?; + writer.write(self.data)?; + Ok(3 + data_len) + } + 0x100..=U16_MAX => { + let cast = (data_len as u16).to_be_bytes(); + writer.write(&[Format::Ext16.as_byte(), cast[0], cast[1], type_byte])?; + writer.write(self.data)?; + Ok(4 + data_len) + } + 0x10000..=U32_MAX => { + let cast = (data_len as u32).to_be_bytes(); + writer.write(&[ + Format::Ext32.as_byte(), + cast[0], + cast[1], + cast[2], + cast[3], + type_byte, + ])?; + writer.write(self.data)?; + Ok(6 + data_len) + } + _ => Err(encode::Error::InvalidFormat), + } + } +} + +impl<'a> Decode<'a> for ExtensionRef<'a> { + type Value = ExtensionRef<'a>; + + fn decode(buf: &'a [u8]) -> core::result::Result<(Self::Value, &'a [u8]), decode::Error> { + let (format, buf) = Format::decode(buf)?; + match format { + Format::FixExt1 + | Format::FixExt2 + | Format::FixExt4 + | Format::FixExt8 + | Format::FixExt16 + | Format::Ext8 + | Format::Ext16 + | Format::Ext32 => Self::decode_with_format(format, buf), + _ => Err(decode::Error::UnexpectedFormat), + } + } + + fn decode_with_format( + format: Format, + buf: &'a [u8], + ) -> core::result::Result<(Self::Value, &'a [u8]), decode::Error> { + let (ext_type, buf) = buf.split_first().ok_or(decode::Error::EofData)?; + let (len, buf) = match format { + Format::FixExt1 => (1, buf), + Format::FixExt2 => (2, buf), + Format::FixExt4 => (4, buf), + Format::FixExt8 => (8, buf), + Format::FixExt16 => (16, buf), + Format::Ext8 => NbyteReader::<1>::read(buf)?, + Format::Ext16 => NbyteReader::<2>::read(buf)?, + Format::Ext32 => NbyteReader::<4>::read(buf)?, + _ => return Err(decode::Error::UnexpectedFormat), + }; + let (data, rest) = buf.split_at_checked(len).ok_or(decode::Error::EofData)?; + let ext = ExtensionRef { + r#type: (*ext_type) as i8, + data, + }; + Ok((ext, rest)) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct FixedExtension { + pub r#type: i8, + len: usize, + data: [u8; N], +} + +impl FixedExtension { + pub fn new(r#type: i8, data: &[u8]) -> Option { + if data.len() > N { + return None; + } + let mut buf = [0u8; N]; + buf[..data.len()].copy_from_slice(data); + Some(Self { + r#type, + len: data.len(), + data: buf, + }) + } + + pub fn new_fixed(r#type: i8, data: [u8; N]) -> Self { + Self { + r#type, + len: N, + data, + } + } + + pub fn as_ref(&self) -> ExtensionRef<'_> { + ExtensionRef { + r#type: self.r#type, + data: &self.data[..self.len], + } + } + + pub fn len(&self) -> usize { + self.len + } + + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + pub fn data(&self) -> &[u8] { + &self.data[..self.len] + } +} + +impl Encode for FixedExtension { + fn encode(&self, writer: &mut W) -> core::result::Result> { + self.as_ref().encode(writer) + } +} + +impl<'a, const N: usize> Decode<'a> for FixedExtension { + type Value = FixedExtension; + + fn decode(buf: &'a [u8]) -> core::result::Result<(Self::Value, &'a [u8]), decode::Error> { + let (ext, rest) = ExtensionRef::decode(buf)?; + if ext.data.len() > N { + return Err(decode::Error::InvalidData); + } + let mut buf_arr = [0u8; N]; + buf_arr[..ext.data.len()].copy_from_slice(ext.data); + Ok(( + FixedExtension { + r#type: ext.r#type, + len: ext.data.len(), + data: buf_arr, + }, + rest, + )) + } + + fn decode_with_format( + format: Format, + buf: &'a [u8], + ) -> core::result::Result<(Self::Value, &'a [u8]), decode::Error> { + let (ext, rest) = ExtensionRef::decode_with_format(format, buf)?; + if ext.data.len() > N { + return Err(decode::Error::InvalidData); + } + let mut buf_arr = [0u8; N]; + buf_arr[..ext.data.len()].copy_from_slice(ext.data); + Ok(( + FixedExtension { + r#type: ext.r#type, + len: ext.data.len(), + data: buf_arr, + }, + rest, + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + + #[rstest] + #[case(0xd4,123,[0x12])] + #[case(0xd5,123,[0x12,0x34])] + #[case(0xd6,123,[0x12,0x34,0x56,0x78])] + #[case(0xd7,123,[0x12;8])] + #[case(0xd8,123,[0x12;16])] + fn encode_ext_fixed>(#[case] marker: u8, #[case] ty: i8, #[case] data: D) { + let expected = marker + .to_be_bytes() + .iter() + .chain(ty.to_be_bytes().iter()) + .chain(data.as_ref()) + .cloned() + .collect::>(); + + let encoder = ExtensionRef::new(ty, data.as_ref()); + + let mut buf = vec![]; + let n = encoder.encode(&mut buf).unwrap(); + + assert_eq!(&buf, &expected); + assert_eq!(n, expected.len()); + } + + #[rstest] + #[case(0xc7_u8.to_be_bytes(),123,5u8.to_be_bytes(),[0x12;5])] + #[case(0xc8_u8.to_be_bytes(),123,65535_u16.to_be_bytes(),[0x34;65535])] + #[case(0xc9_u8.to_be_bytes(),123,65536_u32.to_be_bytes(),[0x56;65536])] + fn encode_ext_sized, S: AsRef<[u8]>, D: AsRef<[u8]>>( + #[case] marker: M, + #[case] ty: i8, + #[case] size: S, + #[case] data: D, + ) { + let expected = marker + .as_ref() + .iter() + .chain(size.as_ref()) + .chain(ty.to_be_bytes().iter()) + .chain(data.as_ref()) + .cloned() + .collect::>(); + + let encoder = ExtensionRef::new(ty, data.as_ref()); + + let mut buf = vec![]; + let n = encoder.encode(&mut buf).unwrap(); + + assert_eq!(&buf, &expected); + assert_eq!(n, expected.len()); + } + + const TIMESTAMP32: &[u8] = &[0xd6, 0xff, 0x62, 0x15, 0x62, 0x1e]; + + #[test] + fn decode_fix_ext4() { + let (ext, rest) = ExtensionRef::decode(TIMESTAMP32).unwrap(); + let expect_type = -1; + let expect_data = 1645568542; + assert_eq!(ext.r#type, expect_type); + let data_u32 = u32::from_be_bytes(ext.data.try_into().unwrap()); + assert_eq!(data_u32, expect_data); + assert_eq!(rest.len(), 0); + } + + #[test] + fn fixed_extension_roundtrip() { + let data = [1u8, 2, 3, 4]; + let ext = FixedExtension::<8>::new(5, &data).unwrap(); + let mut buf = vec![]; + ext.encode(&mut buf).unwrap(); + let (decoded, rest) = FixedExtension::<8>::decode(&buf).unwrap(); + assert_eq!(decoded.r#type, 5); + assert_eq!(decoded.data(), &data); + assert!(rest.is_empty()); + } +} diff --git a/messagepack-core/src/lib.rs b/messagepack-core/src/lib.rs index 9df132b..1c7f665 100644 --- a/messagepack-core/src/lib.rs +++ b/messagepack-core/src/lib.rs @@ -8,6 +8,7 @@ pub mod encode; mod formats; pub mod io; pub mod timestamp; +pub mod extension; pub use decode::Decode; pub use encode::Encode; diff --git a/messagepack-serde/src/value/extension.rs b/messagepack-serde/src/value/extension.rs index 15c304b..ac56a4c 100644 --- a/messagepack-serde/src/value/extension.rs +++ b/messagepack-serde/src/value/extension.rs @@ -1,4 +1,4 @@ -use messagepack_core::{Format, encode::ExtensionEncoder, io::IoWrite}; +use messagepack_core::{Format, extension::ExtensionRef as CoreExtensionRef, io::IoWrite}; use serde::{ Deserialize, Serialize, Serializer, de::Visitor, @@ -265,9 +265,9 @@ impl ser::Serialize for ExtInner<'_> { where S: Serializer, { - let encoder = ExtensionEncoder::new(self.kind, self.data); + let encoder = CoreExtensionRef::new(self.kind, self.data); let format = encoder - .to_format::<()>() + .to_format::() .map_err(|_| ser::Error::custom("Invalid data length"))?; let mut seq = serializer.serialize_seq(Some(4))?; From d71cc4e6ff57e5ffcd1e78deb908354d50b676cb Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Sat, 6 Sep 2025 05:06:37 +0000 Subject: [PATCH 16/57] chore: remove serde_bytes --- messagepack-serde/Cargo.toml | 2 +- messagepack-serde/src/value/extension.rs | 49 ++++++++++++------------ 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/messagepack-serde/Cargo.toml b/messagepack-serde/Cargo.toml index 7b5ce59..6b2bb4c 100644 --- a/messagepack-serde/Cargo.toml +++ b/messagepack-serde/Cargo.toml @@ -13,10 +13,10 @@ all-features = true [dependencies] messagepack-core = { workspace = true } serde = { version = "1.0", default-features = false, features = ["derive"] } -serde_bytes = { version = "0.11" } num-traits = { workspace = true } [dev-dependencies] +serde_bytes = { version = "0.11" } rstest = { workspace = true } [features] diff --git a/messagepack-serde/src/value/extension.rs b/messagepack-serde/src/value/extension.rs index ac56a4c..86ef3e5 100644 --- a/messagepack-serde/src/value/extension.rs +++ b/messagepack-serde/src/value/extension.rs @@ -255,6 +255,16 @@ where } } +struct Bytes<'a>(pub &'a [u8]); +impl ser::Serialize for Bytes<'_> { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_bytes(self.0) + } +} + struct ExtInner<'a> { kind: i8, data: &'a [u8], @@ -272,42 +282,31 @@ impl ser::Serialize for ExtInner<'_> { let mut seq = serializer.serialize_seq(Some(4))?; - seq.serialize_element(serde_bytes::Bytes::new(&format.as_slice()))?; - - const EMPTY: &[u8] = &[]; + seq.serialize_element(&Bytes(&format.as_slice()))?; match format { - messagepack_core::Format::FixExt1 => { - seq.serialize_element(serde_bytes::Bytes::new(EMPTY)) - } - messagepack_core::Format::FixExt2 => { - seq.serialize_element(serde_bytes::Bytes::new(EMPTY)) - } - messagepack_core::Format::FixExt4 => { - seq.serialize_element(serde_bytes::Bytes::new(EMPTY)) - } - messagepack_core::Format::FixExt8 => { - seq.serialize_element(serde_bytes::Bytes::new(EMPTY)) - } - messagepack_core::Format::FixExt16 => { - seq.serialize_element(serde_bytes::Bytes::new(EMPTY)) - } + messagepack_core::Format::FixExt1 + | messagepack_core::Format::FixExt2 + | messagepack_core::Format::FixExt4 + | messagepack_core::Format::FixExt8 + | messagepack_core::Format::FixExt16 => {} + messagepack_core::Format::Ext8 => { let len = (self.data.len() as u8).to_be_bytes(); - seq.serialize_element(serde_bytes::Bytes::new(&len)) + seq.serialize_element(&Bytes(&len))?; } messagepack_core::Format::Ext16 => { let len = (self.data.len() as u16).to_be_bytes(); - seq.serialize_element(serde_bytes::Bytes::new(&len)) + seq.serialize_element(&Bytes(&len))?; } messagepack_core::Format::Ext32 => { let len = (self.data.len() as u32).to_be_bytes(); - seq.serialize_element(serde_bytes::Bytes::new(&len)) + seq.serialize_element(&Bytes(&len))?; } - _ => unreachable!(), - }?; - seq.serialize_element(serde_bytes::Bytes::new(&self.kind.to_be_bytes()))?; - seq.serialize_element(serde_bytes::Bytes::new(self.data))?; + _ => return Err(ser::Error::custom("unexpected format")), + }; + seq.serialize_element(&Bytes(&self.kind.to_be_bytes()))?; + seq.serialize_element(&Bytes(self.data))?; seq.end() } From 8dcfc8e5db3cedea6f2d966b83539232296ef97e Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Sat, 6 Sep 2025 05:09:23 +0000 Subject: [PATCH 17/57] chore: cargo fmt --- messagepack-core/src/encode/bin.rs | 8 +------- messagepack-core/src/encode/map.rs | 8 +------- messagepack-core/src/lib.rs | 2 +- messagepack-serde/src/value/value_.rs | 1 - 4 files changed, 3 insertions(+), 16 deletions(-) diff --git a/messagepack-core/src/encode/bin.rs b/messagepack-core/src/encode/bin.rs index 8283cfd..806093f 100644 --- a/messagepack-core/src/encode/bin.rs +++ b/messagepack-core/src/encode/bin.rs @@ -27,13 +27,7 @@ impl Encode for BinaryEncoder<'_> { } 0x10000..=0xffffffff => { let cast = (self_len as u32).to_be_bytes(); - writer.write(&[ - Format::Bin32.as_byte(), - cast[0], - cast[1], - cast[2], - cast[3], - ])?; + writer.write(&[Format::Bin32.as_byte(), cast[0], cast[1], cast[2], cast[3]])?; Ok(5) } diff --git a/messagepack-core/src/encode/map.rs b/messagepack-core/src/encode/map.rs index 2e3ffd4..64c90b2 100644 --- a/messagepack-core/src/encode/map.rs +++ b/messagepack-core/src/encode/map.rs @@ -49,13 +49,7 @@ impl Encode for MapFormatEncoder { } 0x10000..=0xffffffff => { let cast = (self.0 as u32).to_be_bytes(); - writer.write(&[ - Format::Map32.as_byte(), - cast[0], - cast[1], - cast[2], - cast[3], - ])?; + writer.write(&[Format::Map32.as_byte(), cast[0], cast[1], cast[2], cast[3]])?; Ok(5) } diff --git a/messagepack-core/src/lib.rs b/messagepack-core/src/lib.rs index 1c7f665..02cf5f8 100644 --- a/messagepack-core/src/lib.rs +++ b/messagepack-core/src/lib.rs @@ -5,10 +5,10 @@ pub mod decode; pub mod encode; +pub mod extension; mod formats; pub mod io; pub mod timestamp; -pub mod extension; pub use decode::Decode; pub use encode::Encode; diff --git a/messagepack-serde/src/value/value_.rs b/messagepack-serde/src/value/value_.rs index be067df..d39f7df 100644 --- a/messagepack-serde/src/value/value_.rs +++ b/messagepack-serde/src/value/value_.rs @@ -164,7 +164,6 @@ impl<'de> serde::Deserialize<'de> for ValueRef<'de> { A: serde::de::SeqAccess<'de>, { let mut buf = Vec::new(); - while let Some(v) = seq.next_element::()? { buf.push(v); From 23478bf5d5ce8d94af073e3d531d094c346dad20 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 6 Sep 2025 05:14:49 +0000 Subject: [PATCH 18/57] chore(deps): bump codecov/codecov-action from 5.4.3 to 5.5.0 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 5.4.3 to 5.5.0. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/18283e04ce6e62d37312384ff67231eb8fd56d24...fdcc8476540edceab3de004e990f80d881c6cc00) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-version: 5.5.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/coverage.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index 3252e9a..56f362f 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -41,7 +41,7 @@ jobs: - name: Run test coverage run: | cargo llvm-cov --all-features --workspace --lcov --output-path lcov.info - - uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5.4.3 + - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 with: files: ./lcov.info env: From 0d9261dab058626bde10f647deacb0410fd465f5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 6 Sep 2025 05:16:31 +0000 Subject: [PATCH 19/57] chore(deps): bump CodSpeedHQ/action from 3.6.1 to 3.8.1 Bumps [CodSpeedHQ/action](https://github.com/codspeedhq/action) from 3.6.1 to 3.8.1. - [Release notes](https://github.com/codspeedhq/action/releases) - [Changelog](https://github.com/CodSpeedHQ/action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codspeedhq/action/compare/ad71f92d9429aabfcf70738be9defdbbfc7b75e2...76578c2a7ddd928664caa737f0e962e3085d4e7c) --- updated-dependencies: - dependency-name: CodSpeedHQ/action dependency-version: 3.8.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/codspeed.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codspeed.yaml b/.github/workflows/codspeed.yaml index 6677d44..249007f 100644 --- a/.github/workflows/codspeed.yaml +++ b/.github/workflows/codspeed.yaml @@ -42,7 +42,7 @@ jobs: run: cargo codspeed build - name: Run the benchmarks - uses: CodSpeedHQ/action@ad71f92d9429aabfcf70738be9defdbbfc7b75e2 # v3.6.1 + uses: CodSpeedHQ/action@6eeb021fd0f305388292348b775d96d95253adf4 # v4.0.0 with: run: cargo codspeed run token: ${{ secrets.CODSPEED_TOKEN }} \ No newline at end of file From cca00aabb07430e40d515c0fdef21f88f6d8372b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 6 Sep 2025 05:18:29 +0000 Subject: [PATCH 20/57] chore(deps): bump actions/checkout from 4.2.2 to 5.0.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.2.2 to 5.0.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/11bd71901bbe5b1630ceea73d27597364c9af683...08c6903cd8c0fde910a37f88322edcfb5dd907a8) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: 5.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codspeed.yaml | 2 +- .github/workflows/coverage.yaml | 2 +- .github/workflows/pull_request.yaml | 6 +++--- .github/workflows/release.yaml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/codspeed.yaml b/.github/workflows/codspeed.yaml index 249007f..0a6536d 100644 --- a/.github/workflows/codspeed.yaml +++ b/.github/workflows/codspeed.yaml @@ -30,7 +30,7 @@ jobs: name: Run benchmarks runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Setup rust toolchain, cache and cargo-codspeed binary uses: moonrepo/setup-rust@ede6de059f8046a5e236c94046823e2af11ca670 # v1.2.2 diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index 56f362f..5ba96fd 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -31,7 +31,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout PR branch - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Install Rust uses: moonrepo/setup-rust@ede6de059f8046a5e236c94046823e2af11ca670 # v1.2.2 with: diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 50c7c18..8126cc4 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -34,7 +34,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout PR branch - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Install Rust uses: moonrepo/setup-rust@ede6de059f8046a5e236c94046823e2af11ca670 # v1.2.2 with: @@ -48,7 +48,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout PR branch - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Install Rust uses: moonrepo/setup-rust@ede6de059f8046a5e236c94046823e2af11ca670 # v1.2.2 with: @@ -65,7 +65,7 @@ jobs: os: [ubuntu-latest, windows-latest] steps: - name: Checkout PR branch - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Install Rust uses: moonrepo/setup-rust@ede6de059f8046a5e236c94046823e2af11ca670 # v1.2.2 with: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c95d06c..10b27c7 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -12,7 +12,7 @@ jobs: steps: - name: Checkout branch - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Install Rust uses: moonrepo/setup-rust@ede6de059f8046a5e236c94046823e2af11ca670 # v1.2.2 - name: Publish crates.io From 28581f08ea674004ca4ec22acb4ef46899951332 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Sat, 6 Sep 2025 14:10:11 +0000 Subject: [PATCH 21/57] feat: ser/de wire for ext_ref --- messagepack-serde/src/de/mod.rs | 11 ---- messagepack-serde/src/lib.rs | 3 +- messagepack-serde/src/ser/mod.rs | 11 ---- messagepack-serde/src/value/extension.rs | 77 +++++++++++------------- messagepack-serde/src/value/mod.rs | 2 +- 5 files changed, 39 insertions(+), 65 deletions(-) diff --git a/messagepack-serde/src/de/mod.rs b/messagepack-serde/src/de/mod.rs index 4a268fe..f08d03a 100644 --- a/messagepack-serde/src/de/mod.rs +++ b/messagepack-serde/src/de/mod.rs @@ -554,15 +554,4 @@ mod tests { let decoded = from_slice::(buf.as_ref()).unwrap(); assert_eq!(decoded, expected); } - - #[rstest] - fn decode_extension() { - use crate::value::extension::ExtensionRef; - - let buf: &[u8] = &[0xd4, 0x7b, 0x12]; - - let ext = from_slice::>(buf).unwrap(); - assert_eq!(ext.kind, 123); - assert_eq!(ext.data, [0x12_u8]) - } } diff --git a/messagepack-serde/src/lib.rs b/messagepack-serde/src/lib.rs index 0c1c8a2..98bb5f9 100644 --- a/messagepack-serde/src/lib.rs +++ b/messagepack-serde/src/lib.rs @@ -19,6 +19,7 @@ pub use de::{from_reader, from_reader_with_config}; #[cfg(feature = "std")] pub use ser::{to_vec, to_vec_with_config, to_writer, to_writer_with_config}; +pub use value::Number; +pub use value::ext_ref; #[cfg(feature = "alloc")] pub use value::ValueRef; -pub use value::{ExtensionRef, Number}; diff --git a/messagepack-serde/src/ser/mod.rs b/messagepack-serde/src/ser/mod.rs index e0e1323..15f36f6 100644 --- a/messagepack-serde/src/ser/mod.rs +++ b/messagepack-serde/src/ser/mod.rs @@ -549,17 +549,6 @@ mod tests { assert_eq!(buf[..len], [0xe0]); } - #[test] - fn encode_extension() { - use crate::value::extension::ExtensionRef; - let kind: i8 = 123; - let ext = ExtensionRef::new(kind, &[0x12]); - let buf = &mut [0_u8; 3]; - - let len = to_slice(&ext, buf).unwrap(); - assert_eq!(buf[..len], [0xd4, kind.to_be_bytes()[0], 0x12]); - } - #[cfg(feature = "std")] #[test] fn encode_with_writer() { diff --git a/messagepack-serde/src/value/extension.rs b/messagepack-serde/src/value/extension.rs index 86ef3e5..bbf8be8 100644 --- a/messagepack-serde/src/value/extension.rs +++ b/messagepack-serde/src/value/extension.rs @@ -9,19 +9,6 @@ use crate::ser::{CoreError, Error}; pub(crate) const EXTENSION_STRUCT_NAME: &str = "$__MSGPACK_EXTENSION_STRUCT"; -/// Represents `ext` format. This is also available with `no_std` to borrow data. -#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord)] -pub struct ExtensionRef<'a> { - pub kind: i8, - pub data: &'a [u8], -} - -impl<'a> ExtensionRef<'a> { - pub fn new(kind: i8, data: &'a [u8]) -> Self { - Self { kind, data } - } -} - pub(crate) struct SerializeExt<'a, W> { writer: &'a mut W, length: &'a mut usize, @@ -312,22 +299,8 @@ impl ser::Serialize for ExtInner<'_> { } } -impl ser::Serialize for ExtensionRef<'_> { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_newtype_struct( - EXTENSION_STRUCT_NAME, - &ExtInner { - kind: self.kind, - data: self.data, - }, - ) - } -} - pub(crate) struct DeserializeExt<'de> { + format: Format, data_len: usize, pub(crate) input: &'de [u8], } @@ -370,6 +343,7 @@ impl<'de> DeserializeExt<'de> { _ => return Err(messagepack_core::decode::Error::UnexpectedFormat.into()), }; Ok(DeserializeExt { + format, data_len, input: rest, }) @@ -447,15 +421,35 @@ impl<'de> serde::de::SeqAccess<'de> for &mut DeserializeExt<'de> { } } -impl<'de> Deserialize<'de> for ExtensionRef<'de> { - fn deserialize(deserializer: D) -> Result +pub mod ext_ref { + use super::*; + + pub fn serialize( + ext: &messagepack_core::extension::ExtensionRef<'_>, + serializer: S, + ) -> Result + where + S: serde::Serializer, + { + serializer.serialize_newtype_struct( + EXTENSION_STRUCT_NAME, + &ExtInner { + kind: ext.r#type, + data: ext.data, + }, + ) + } + + pub fn deserialize<'de, D>( + deserializer: D, + ) -> Result, D::Error> where D: serde::Deserializer<'de>, { struct ExtensionVisitor; impl<'de> Visitor<'de> for ExtensionVisitor { - type Value = ExtensionRef<'de>; + type Value = messagepack_core::extension::ExtensionRef<'de>; fn expecting(&self, formatter: &mut core::fmt::Formatter) -> core::fmt::Result { formatter.write_str("expect extension") } @@ -479,7 +473,7 @@ impl<'de> Deserialize<'de> for ExtensionRef<'de> { .next_element::<&[u8]>()? .ok_or(serde::de::Error::custom("expect [u8]"))?; - Ok(ExtensionRef::new(kind, data)) + Ok(messagepack_core::extension::ExtensionRef::new(kind, data)) } } deserializer.deserialize_any(ExtensionVisitor) @@ -489,21 +483,22 @@ impl<'de> Deserialize<'de> for ExtensionRef<'de> { #[cfg(test)] mod tests { use super::*; - use messagepack_core::SliceWriter; + use messagepack_core::extension::ExtensionRef; use rstest::rstest; + #[derive(Debug, Serialize, Deserialize)] + struct Wrap<'a>( + #[serde(with = "ext_ref", borrow)] messagepack_core::extension::ExtensionRef<'a>, + ); + #[rstest] fn encode_ext() { let mut buf = [0_u8; 3]; - let mut writer = SliceWriter::from_slice(&mut buf); - let mut length = 0; - let mut ser = SerializeExt::new(&mut writer, &mut length); let kind: i8 = 123; - let ext = ExtensionRef::new(kind, &[0x12]); - - ext.serialize(&mut ser).unwrap(); + let ext = Wrap(ExtensionRef::new(kind, &[0x12])); + let length = crate::to_slice(&ext, &mut buf).unwrap(); assert_eq!(length, 3); assert_eq!(buf, [0xd4, kind.to_be_bytes()[0], 0x12]); @@ -513,8 +508,8 @@ mod tests { fn decode_ext() { let buf = [0xd6, 0xff, 0x00, 0x00, 0x00, 0x00]; // timestamp ext type - let ext = crate::from_slice::(&buf).unwrap(); - assert_eq!(ext.kind, -1); + let ext = crate::from_slice::>(&buf).unwrap().0; + assert_eq!(ext.r#type, -1); let seconds = u32::from_be_bytes(ext.data.try_into().unwrap()); assert_eq!(seconds, 0); } diff --git a/messagepack-serde/src/value/mod.rs b/messagepack-serde/src/value/mod.rs index 76abca9..239eebc 100644 --- a/messagepack-serde/src/value/mod.rs +++ b/messagepack-serde/src/value/mod.rs @@ -4,6 +4,6 @@ pub(crate) mod value_; pub use value_::ValueRef; pub(crate) mod extension; -pub use extension::ExtensionRef; +pub use extension::ext_ref; pub(crate) mod number; pub use number::Number; From 22f1eb18655c24781048b90732575a5f9c8ce967 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Sat, 6 Sep 2025 14:10:24 +0000 Subject: [PATCH 22/57] chore: fix conf --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 991f8ac..b402c6b 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -11,7 +11,7 @@ "files.watcherExclude": { "**/target/**": true }, - "rust-analyzer.checkOnSave.command": "clippy" + "rust-analyzer.checkOnSave": true }, // Add the IDs of extensions you want installed when the container is created. "extensions": [ From 1bde1fb5df275201763f85c73f9e908ce432d634 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Sat, 6 Sep 2025 14:42:12 +0000 Subject: [PATCH 23/57] feat: add ext_fixed --- messagepack-serde/src/lib.rs | 2 +- messagepack-serde/src/value/extension.rs | 71 ++++++++++++++++++++++-- messagepack-serde/src/value/mod.rs | 2 +- 3 files changed, 67 insertions(+), 8 deletions(-) diff --git a/messagepack-serde/src/lib.rs b/messagepack-serde/src/lib.rs index 98bb5f9..5869c90 100644 --- a/messagepack-serde/src/lib.rs +++ b/messagepack-serde/src/lib.rs @@ -20,6 +20,6 @@ pub use de::{from_reader, from_reader_with_config}; pub use ser::{to_vec, to_vec_with_config, to_writer, to_writer_with_config}; pub use value::Number; -pub use value::ext_ref; #[cfg(feature = "alloc")] pub use value::ValueRef; +pub use value::{ext_fixed, ext_ref}; diff --git a/messagepack-serde/src/value/extension.rs b/messagepack-serde/src/value/extension.rs index bbf8be8..5cf2cea 100644 --- a/messagepack-serde/src/value/extension.rs +++ b/messagepack-serde/src/value/extension.rs @@ -480,24 +480,51 @@ pub mod ext_ref { } } +pub mod ext_fixed { + use serde::de; + + pub fn serialize( + ext: &messagepack_core::extension::FixedExtension, + serializer: S, + ) -> Result + where + S: serde::Serializer, + { + super::ext_ref::serialize(&ext.as_ref(), serializer) + } + + pub fn deserialize<'de, const N: usize, D>( + deserializer: D, + ) -> Result, D::Error> + where + D: serde::Deserializer<'de>, + { + let r = super::ext_ref::deserialize(deserializer)?; + + let ext = messagepack_core::extension::FixedExtension::new(r.r#type, r.data) + .ok_or_else(|| de::Error::custom("extension length is too long"))?; + Ok(ext) + } +} + #[cfg(test)] mod tests { use super::*; - use messagepack_core::extension::ExtensionRef; + use messagepack_core::extension::{ExtensionRef, FixedExtension}; use rstest::rstest; #[derive(Debug, Serialize, Deserialize)] - struct Wrap<'a>( + struct WrapRef<'a>( #[serde(with = "ext_ref", borrow)] messagepack_core::extension::ExtensionRef<'a>, ); #[rstest] - fn encode_ext() { + fn encode_ext_ref() { let mut buf = [0_u8; 3]; let kind: i8 = 123; - let ext = Wrap(ExtensionRef::new(kind, &[0x12])); + let ext = WrapRef(ExtensionRef::new(kind, &[0x12])); let length = crate::to_slice(&ext, &mut buf).unwrap(); assert_eq!(length, 3); @@ -505,12 +532,44 @@ mod tests { } #[rstest] - fn decode_ext() { + fn decode_ext_ref() { let buf = [0xd6, 0xff, 0x00, 0x00, 0x00, 0x00]; // timestamp ext type - let ext = crate::from_slice::>(&buf).unwrap().0; + let ext = crate::from_slice::>(&buf).unwrap().0; assert_eq!(ext.r#type, -1); let seconds = u32::from_be_bytes(ext.data.try_into().unwrap()); assert_eq!(seconds, 0); } + + #[derive(Debug, Serialize, Deserialize)] + struct WrapFixed( + #[serde(with = "ext_fixed")] messagepack_core::extension::FixedExtension, + ); + + #[rstest] + fn encode_ext_fixed() { + let mut buf = [0u8; 3]; + let kind: i8 = 123; + + let ext = WrapFixed(FixedExtension::new_fixed(kind, [0x12])); + let length = crate::to_slice(&ext, &mut buf).unwrap(); + + assert_eq!(length, 3); + assert_eq!(buf, [0xd4, kind.to_be_bytes()[0], 0x12]); + } + + const TIMESTAMP32: &[u8] = &[0xd6, 0xff, 0x00, 0x00, 0x00, 0x00]; + + #[rstest] + fn decode_ext_fixed_bigger_will_success() { + let ext = crate::from_slice::>(TIMESTAMP32).unwrap().0; + assert_eq!(ext.r#type, -1); + assert_eq!(ext.data(), &TIMESTAMP32[2..]) + } + + #[rstest] + #[should_panic] + fn decode_ext_fixed_smaller_will_failed() { + let _ = crate::from_slice::>(TIMESTAMP32).unwrap(); + } } diff --git a/messagepack-serde/src/value/mod.rs b/messagepack-serde/src/value/mod.rs index 239eebc..eec10b0 100644 --- a/messagepack-serde/src/value/mod.rs +++ b/messagepack-serde/src/value/mod.rs @@ -4,6 +4,6 @@ pub(crate) mod value_; pub use value_::ValueRef; pub(crate) mod extension; -pub use extension::ext_ref; +pub use extension::{ext_fixed, ext_ref}; pub(crate) mod number; pub use number::Number; From 24ee78d7505a0864798fc5bf8500bfeb96de883f Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Sat, 6 Sep 2025 14:46:44 +0000 Subject: [PATCH 24/57] refactor: fix visibility --- messagepack-core/src/extension.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/messagepack-core/src/extension.rs b/messagepack-core/src/extension.rs index 941c08b..3ef2ba7 100644 --- a/messagepack-core/src/extension.rs +++ b/messagepack-core/src/extension.rs @@ -2,9 +2,9 @@ use crate::decode::{self, NbyteReader}; use crate::encode; use crate::{Decode, Encode, formats::Format, io::IoWrite}; -pub const U8_MAX: usize = u8::MAX as usize; -pub const U16_MAX: usize = u16::MAX as usize; -pub const U32_MAX: usize = u32::MAX as usize; +const U8_MAX: usize = u8::MAX as usize; +const U16_MAX: usize = u16::MAX as usize; +const U32_MAX: usize = u32::MAX as usize; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] pub struct ExtensionRef<'a> { From f155e4f591135e51282dedad9124571f7cb4ca35 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Sat, 6 Sep 2025 14:47:15 +0000 Subject: [PATCH 25/57] refactor: write length --- messagepack-serde/src/ser/mod.rs | 6 ++++-- messagepack-serde/src/value/extension.rs | 12 ++++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/messagepack-serde/src/ser/mod.rs b/messagepack-serde/src/ser/mod.rs index 15f36f6..0714f0c 100644 --- a/messagepack-serde/src/ser/mod.rs +++ b/messagepack-serde/src/ser/mod.rs @@ -250,8 +250,10 @@ where { match name { EXTENSION_STRUCT_NAME => { - let mut ser = SerializeExt::new(self.writer, &mut self.current_length); - value.serialize(&mut ser) + let mut ser = SerializeExt::new(self.writer); + value.serialize(&mut ser)?; + self.current_length += ser.length(); + Ok(()) } _ => value.serialize(self.as_mut()), } diff --git a/messagepack-serde/src/value/extension.rs b/messagepack-serde/src/value/extension.rs index 5cf2cea..231c072 100644 --- a/messagepack-serde/src/value/extension.rs +++ b/messagepack-serde/src/value/extension.rs @@ -11,7 +11,7 @@ pub(crate) const EXTENSION_STRUCT_NAME: &str = "$__MSGPACK_EXTENSION_STRUCT"; pub(crate) struct SerializeExt<'a, W> { writer: &'a mut W, - length: &'a mut usize, + length: usize, } impl AsMut for SerializeExt<'_, W> { @@ -21,8 +21,12 @@ impl AsMut for SerializeExt<'_, W> { } impl<'a, W> SerializeExt<'a, W> { - pub fn new(writer: &'a mut W, length: &'a mut usize) -> Self { - Self { writer, length } + pub fn new(writer: &'a mut W) -> Self { + Self { writer, length: 0 } + } + + pub(crate) fn length(&self) -> usize { + self.length } } @@ -109,7 +113,7 @@ where fn serialize_bytes(self, v: &[u8]) -> Result { self.writer.write(v).map_err(CoreError::Io)?; - *self.length += v.len(); + self.length += v.len(); Ok(()) } From 418f07a50bfe4ee107e3f78c8e0c879570825814 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Sat, 6 Sep 2025 14:54:01 +0000 Subject: [PATCH 26/57] fix: lint error --- messagepack-serde/src/value/extension.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/messagepack-serde/src/value/extension.rs b/messagepack-serde/src/value/extension.rs index 231c072..d4ce2ad 100644 --- a/messagepack-serde/src/value/extension.rs +++ b/messagepack-serde/src/value/extension.rs @@ -304,7 +304,6 @@ impl ser::Serialize for ExtInner<'_> { } pub(crate) struct DeserializeExt<'de> { - format: Format, data_len: usize, pub(crate) input: &'de [u8], } @@ -347,7 +346,6 @@ impl<'de> DeserializeExt<'de> { _ => return Err(messagepack_core::decode::Error::UnexpectedFormat.into()), }; Ok(DeserializeExt { - format, data_len, input: rest, }) From a630f59b3eac8e5177d4e90f63edc7fca8bb96fd Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Sat, 6 Sep 2025 14:58:39 +0000 Subject: [PATCH 27/57] fix: lint error --- messagepack-serde/src/value/extension.rs | 3 ++- messagepack-serde/src/value/value_.rs | 11 +++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/messagepack-serde/src/value/extension.rs b/messagepack-serde/src/value/extension.rs index d4ce2ad..a8d1370 100644 --- a/messagepack-serde/src/value/extension.rs +++ b/messagepack-serde/src/value/extension.rs @@ -1,6 +1,6 @@ use messagepack_core::{Format, extension::ExtensionRef as CoreExtensionRef, io::IoWrite}; use serde::{ - Deserialize, Serialize, Serializer, + Serialize, Serializer, de::Visitor, ser::{self, SerializeSeq}, }; @@ -514,6 +514,7 @@ mod tests { use super::*; use messagepack_core::extension::{ExtensionRef, FixedExtension}; use rstest::rstest; + use serde::{Deserialize, Serialize}; #[derive(Debug, Serialize, Deserialize)] struct WrapRef<'a>( diff --git a/messagepack-serde/src/value/value_.rs b/messagepack-serde/src/value/value_.rs index d39f7df..b27be85 100644 --- a/messagepack-serde/src/value/value_.rs +++ b/messagepack-serde/src/value/value_.rs @@ -1,6 +1,7 @@ -use super::{extension::ExtensionRef, number::Number}; +use super::number::Number; use alloc::vec::Vec; -use serde::{Deserialize, de::Visitor, ser::SerializeMap}; +use messagepack_core::extension::ExtensionRef; +use serde::{de::Visitor, ser::SerializeMap}; /// Represents any messagepack value. `alloc` needed. #[derive(Debug, Clone, PartialEq, PartialOrd)] @@ -72,7 +73,9 @@ impl serde::Serialize for ValueRef<'_> { ValueRef::Nil => serializer.serialize_none(), ValueRef::Bool(v) => serializer.serialize_bool(*v), ValueRef::Bin(items) => (*items).serialize(serializer), - ValueRef::Extension(extension_ref) => extension_ref.serialize(serializer), + ValueRef::Extension(extension_ref) => { + super::ext_ref::serialize(extension_ref, serializer) + } ValueRef::Number(number) => number.serialize(serializer), ValueRef::String(s) => serializer.serialize_str(s), ValueRef::Array(value_refs) => (*value_refs).serialize(serializer), @@ -155,7 +158,7 @@ impl<'de> serde::Deserialize<'de> for ValueRef<'de> { where D: serde::Deserializer<'de>, { - let ext = ExtensionRef::deserialize(deserializer)?; + let ext = super::ext_ref::deserialize(deserializer)?; Ok(ValueRef::Extension(ext)) } From 97d4b185136891ceb9d62ef9f9fc55671d0acf61 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Sat, 6 Sep 2025 15:03:17 +0000 Subject: [PATCH 28/57] fix: codspeed --- .github/workflows/codspeed.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/codspeed.yaml b/.github/workflows/codspeed.yaml index 0a6536d..13b6d1a 100644 --- a/.github/workflows/codspeed.yaml +++ b/.github/workflows/codspeed.yaml @@ -44,5 +44,6 @@ jobs: - name: Run the benchmarks uses: CodSpeedHQ/action@6eeb021fd0f305388292348b775d96d95253adf4 # v4.0.0 with: + mode: "instrumentation" run: cargo codspeed run token: ${{ secrets.CODSPEED_TOKEN }} \ No newline at end of file From cb43b654fc47c308268389d7632ece80973ac07f Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Sat, 6 Sep 2025 17:08:56 +0000 Subject: [PATCH 29/57] refactor: forward to any --- messagepack-serde/src/de/mod.rs | 246 +---------------------- messagepack-serde/src/value/extension.rs | 2 +- 2 files changed, 7 insertions(+), 241 deletions(-) diff --git a/messagepack-serde/src/de/mod.rs b/messagepack-serde/src/de/mod.rs index f08d03a..13f5f3f 100644 --- a/messagepack-serde/src/de/mod.rs +++ b/messagepack-serde/src/de/mod.rs @@ -5,13 +5,11 @@ mod seq; pub use error::{CoreError, Error}; use crate::value::extension::DeserializeExt; -use messagepack_core::{ - Decode, Format, - decode::{NbyteReader, NilDecoder}, -}; +use messagepack_core::{Decode, Format, decode::NbyteReader}; use serde::{ Deserialize, de::{self, IntoDeserializer}, + forward_to_deserialize_any, }; #[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)] @@ -205,230 +203,6 @@ impl<'de> de::Deserializer<'de> for &mut Deserializer<'de> { } } - fn deserialize_bool(self, visitor: V) -> Result - where - V: de::Visitor<'de>, - { - let decoded = self.decode::()?; - visitor.visit_bool(decoded) - } - - fn deserialize_i8(self, visitor: V) -> Result - where - V: de::Visitor<'de>, - { - let decoded = self.decode::()?; - visitor.visit_i8(decoded) - } - - fn deserialize_i16(self, visitor: V) -> Result - where - V: de::Visitor<'de>, - { - let decoded = self.decode::()?; - visitor.visit_i16(decoded) - } - - fn deserialize_i32(self, visitor: V) -> Result - where - V: de::Visitor<'de>, - { - let decoded = self.decode::()?; - visitor.visit_i32(decoded) - } - - fn deserialize_i64(self, visitor: V) -> Result - where - V: de::Visitor<'de>, - { - let decoded = self.decode::()?; - visitor.visit_i64(decoded) - } - - fn deserialize_u8(self, visitor: V) -> Result - where - V: de::Visitor<'de>, - { - let decoded = self.decode::()?; - visitor.visit_u8(decoded) - } - - fn deserialize_u16(self, visitor: V) -> Result - where - V: de::Visitor<'de>, - { - let decoded = self.decode::()?; - visitor.visit_u16(decoded) - } - - fn deserialize_u32(self, visitor: V) -> Result - where - V: de::Visitor<'de>, - { - let decoded = self.decode::()?; - visitor.visit_u32(decoded) - } - - fn deserialize_u64(self, visitor: V) -> Result - where - V: de::Visitor<'de>, - { - let decoded = self.decode::()?; - visitor.visit_u64(decoded) - } - - fn deserialize_f32(self, visitor: V) -> Result - where - V: de::Visitor<'de>, - { - let decoded = self.decode::()?; - visitor.visit_f32(decoded) - } - - fn deserialize_f64(self, visitor: V) -> Result - where - V: de::Visitor<'de>, - { - let decoded = self.decode::()?; - visitor.visit_f64(decoded) - } - - fn deserialize_char(self, visitor: V) -> Result - where - V: de::Visitor<'de>, - { - self.deserialize_str(visitor) - } - - fn deserialize_str(self, visitor: V) -> Result - where - V: de::Visitor<'de>, - { - let decoded = self.decode::<&str>()?; - visitor.visit_borrowed_str(decoded) - } - - fn deserialize_string(self, visitor: V) -> Result - where - V: de::Visitor<'de>, - { - self.deserialize_str(visitor) - } - - fn deserialize_bytes(self, visitor: V) -> Result - where - V: de::Visitor<'de>, - { - let decoded = self.decode::<&[u8]>()?; - visitor.visit_borrowed_bytes(decoded) - } - - fn deserialize_byte_buf(self, visitor: V) -> Result - where - V: de::Visitor<'de>, - { - self.deserialize_bytes(visitor) - } - - fn deserialize_option(self, visitor: V) -> Result - where - V: de::Visitor<'de>, - { - let is_null = NilDecoder::decode(self.input).is_ok(); - if is_null { - visitor.visit_none() - } else { - visitor.visit_some(self) - } - } - - fn deserialize_unit(self, visitor: V) -> Result - where - V: de::Visitor<'de>, - { - self.decode::<()>()?; - visitor.visit_unit() - } - - fn deserialize_unit_struct( - self, - _name: &'static str, - visitor: V, - ) -> Result - where - V: de::Visitor<'de>, - { - self.deserialize_unit(visitor) - } - - fn deserialize_newtype_struct( - self, - _name: &'static str, - visitor: V, - ) -> Result - where - V: de::Visitor<'de>, - { - visitor.visit_newtype_struct(self) - } - - fn deserialize_seq(self, visitor: V) -> Result - where - V: de::Visitor<'de>, - { - let (format, rest) = Format::decode(self.input)?; - - let mut des = Deserializer::from_slice(rest); - let val = des.decode_seq_with_format(format, visitor)?; - self.input = des.input; - - Ok(val) - } - - fn deserialize_tuple(self, _len: usize, visitor: V) -> Result - where - V: de::Visitor<'de>, - { - self.deserialize_seq(visitor) - } - - fn deserialize_tuple_struct( - self, - _name: &'static str, - _len: usize, - visitor: V, - ) -> Result - where - V: de::Visitor<'de>, - { - self.deserialize_seq(visitor) - } - - fn deserialize_map(self, visitor: V) -> Result - where - V: de::Visitor<'de>, - { - let (format, rest) = Format::decode(self.input)?; - - let mut des = Deserializer::from_slice(rest); - let val = des.decode_map_with_format(format, visitor)?; - self.input = des.input; - - Ok(val) - } - - fn deserialize_struct( - self, - _name: &'static str, - _fields: &'static [&'static str], - visitor: V, - ) -> Result - where - V: de::Visitor<'de>, - { - self.deserialize_map(visitor) - } - fn deserialize_enum( self, _name: &'static str, @@ -462,18 +236,10 @@ impl<'de> de::Deserializer<'de> for &mut Deserializer<'de> { } } - fn deserialize_identifier(self, visitor: V) -> Result - where - V: de::Visitor<'de>, - { - self.deserialize_str(visitor) - } - - fn deserialize_ignored_any(self, visitor: V) -> Result - where - V: de::Visitor<'de>, - { - self.deserialize_any(visitor) + forward_to_deserialize_any! { + bool i8 i16 i32 i64 i128 u8 u16 u32 u64 u128 f32 f64 char str string + bytes byte_buf option unit unit_struct newtype_struct seq tuple + tuple_struct map struct identifier ignored_any } fn is_human_readable(&self) -> bool { diff --git a/messagepack-serde/src/value/extension.rs b/messagepack-serde/src/value/extension.rs index a8d1370..a7265d3 100644 --- a/messagepack-serde/src/value/extension.rs +++ b/messagepack-serde/src/value/extension.rs @@ -478,7 +478,7 @@ pub mod ext_ref { Ok(messagepack_core::extension::ExtensionRef::new(kind, data)) } } - deserializer.deserialize_any(ExtensionVisitor) + deserializer.deserialize_seq(ExtensionVisitor) } } From 581877e07f31e4a2e2453faac9acbfa1499b672c Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Sat, 6 Sep 2025 17:24:49 +0000 Subject: [PATCH 30/57] fix: decode ext len first --- messagepack-core/src/extension.rs | 66 +++++++++++++++++++++++-------- 1 file changed, 50 insertions(+), 16 deletions(-) diff --git a/messagepack-core/src/extension.rs b/messagepack-core/src/extension.rs index 3ef2ba7..885ae98 100644 --- a/messagepack-core/src/extension.rs +++ b/messagepack-core/src/extension.rs @@ -5,6 +5,8 @@ use crate::{Decode, Encode, formats::Format, io::IoWrite}; const U8_MAX: usize = u8::MAX as usize; const U16_MAX: usize = u16::MAX as usize; const U32_MAX: usize = u32::MAX as usize; +const U8_MAX_PLUS_ONE: usize = U8_MAX + 1; +const U16_MAX_PLUS_ONE: usize = U16_MAX + 1; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] pub struct ExtensionRef<'a> { @@ -24,9 +26,9 @@ impl<'a> ExtensionRef<'a> { 4 => Format::FixExt4, 8 => Format::FixExt8, 16 => Format::FixExt16, - 0..U8_MAX => Format::Ext8, - U8_MAX..U16_MAX => Format::Ext16, - U16_MAX..U32_MAX => Format::Ext32, + 0..=U8_MAX => Format::Ext8, + U8_MAX_PLUS_ONE..=U16_MAX => Format::Ext16, + U16_MAX_PLUS_ONE..=U32_MAX => Format::Ext32, _ => return Err(encode::Error::InvalidFormat), }; Ok(format) @@ -116,7 +118,6 @@ impl<'a> Decode<'a> for ExtensionRef<'a> { format: Format, buf: &'a [u8], ) -> core::result::Result<(Self::Value, &'a [u8]), decode::Error> { - let (ext_type, buf) = buf.split_first().ok_or(decode::Error::EofData)?; let (len, buf) = match format { Format::FixExt1 => (1, buf), Format::FixExt2 => (2, buf), @@ -128,6 +129,7 @@ impl<'a> Decode<'a> for ExtensionRef<'a> { Format::Ext32 => NbyteReader::<4>::read(buf)?, _ => return Err(decode::Error::UnexpectedFormat), }; + let (ext_type, buf) = buf.split_first().ok_or(decode::Error::EofData)?; let (data, rest) = buf.split_at_checked(len).ok_or(decode::Error::EofData)?; let ext = ExtensionRef { r#type: (*ext_type) as i8, @@ -290,20 +292,52 @@ mod tests { assert_eq!(n, expected.len()); } - const TIMESTAMP32: &[u8] = &[0xd6, 0xff, 0x62, 0x15, 0x62, 0x1e]; - - #[test] - fn decode_fix_ext4() { - let (ext, rest) = ExtensionRef::decode(TIMESTAMP32).unwrap(); - let expect_type = -1; - let expect_data = 1645568542; - assert_eq!(ext.r#type, expect_type); - let data_u32 = u32::from_be_bytes(ext.data.try_into().unwrap()); - assert_eq!(data_u32, expect_data); - assert_eq!(rest.len(), 0); + #[rstest] + #[case(Format::FixExt1.as_byte(), 5_i8, [0x12])] + #[case(Format::FixExt2.as_byte(), -1_i8, [0x34, 0x56])] + #[case(Format::FixExt4.as_byte(), 42_i8, [0xde, 0xad, 0xbe, 0xef])] + #[case(Format::FixExt8.as_byte(), -7_i8, [0xAA; 8])] + #[case(Format::FixExt16.as_byte(), 7_i8, [0x55; 16])] + fn decode_ext_fixed>(#[case] marker: u8, #[case] ty: i8, #[case] data: E) { + // Buffer: [FixExtN marker][type][data..] + let buf = core::iter::once(marker) + .chain(core::iter::once(ty as u8)) + .chain(data.as_ref().iter().cloned()) + .collect::>(); + + let (ext, rest) = ExtensionRef::decode(&buf).unwrap(); + assert_eq!(ext.r#type, ty); + assert_eq!(ext.data, data.as_ref()); + assert!(rest.is_empty()); + } + + #[rstest] + #[case(Format::Ext8, 42_i8, 5u8.to_be_bytes(), [0x11;5])] // small: Ext8 + #[case(Format::Ext16, -7_i8, 300u16.to_be_bytes(), [0xAA;300])] // medium: Ext16 (>255) + #[case(Format::Ext32, 7_i8, 70000u32.to_be_bytes(), [0x55;70000])] // large: Ext32 (>65535) + fn decode_ext_sized, D: AsRef<[u8]>>( + #[case] format: Format, + #[case] ty: i8, + #[case] size: S, + #[case] data: D, + ) { + // MessagePack ext variable-length layout: [format][length][type][data] + let buf = format + .as_slice() + .iter() + .chain(size.as_ref()) + .chain(ty.to_be_bytes().iter()) + .chain(data.as_ref()) + .cloned() + .collect::>(); + + let (ext, rest) = ExtensionRef::decode(&buf).unwrap(); + assert_eq!(ext.r#type, ty); + assert_eq!(ext.data, data.as_ref()); + assert!(rest.is_empty()); } - #[test] + #[rstest] fn fixed_extension_roundtrip() { let data = [1u8, 2, 3, 4]; let ext = FixedExtension::<8>::new(5, &data).unwrap(); From 73d8a6438a5b5d0cb6aefaf39e150142e6e58490 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Sat, 6 Sep 2025 17:24:58 +0000 Subject: [PATCH 31/57] chore: change base image --- Dockerfile | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index c89a665..ca31410 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,4 @@ -FROM rust:1.85.0-slim-bookworm - -RUN apt-get update -y && \ - apt-get install -y \ - git - -ARG USERNAME=vscode -ARG GROUPNAME=vscode -ARG UID=1000 -ARG GID=1000 -RUN groupadd -g $GID $GROUPNAME && \ - useradd -m -s /bin/bash -u $UID -g $GID $USERNAME - -USER ${USERNAME} +FROM mcr.microsoft.com/devcontainers/rust:1-1-bookworm # Add completions RUN echo "source /usr/share/bash-completion/completions/git" >> /home/vscode/.bashrc From 7cb1ab4dcd035d26421c34d395f5df9e2b68e9b2 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Sat, 6 Sep 2025 17:33:20 +0000 Subject: [PATCH 32/57] chore: fix var name --- messagepack-core/src/decode/timestamp.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/messagepack-core/src/decode/timestamp.rs b/messagepack-core/src/decode/timestamp.rs index 2f94091..e3b28fd 100644 --- a/messagepack-core/src/decode/timestamp.rs +++ b/messagepack-core/src/decode/timestamp.rs @@ -70,8 +70,8 @@ impl<'a> Decode<'a> for Timestamp96 { Format::Ext8 => NbyteReader::<1>::read(buf)?, _ => return Err(Error::UnexpectedFormat), }; - const TIMESTAMP64_DATA_LENGTH: usize = 12; - if len != TIMESTAMP64_DATA_LENGTH { + const TIMESTAMP96_DATA_LENGTH: usize = 12; + if len != TIMESTAMP96_DATA_LENGTH { return Err(Error::InvalidData); } From 75eabd68c1a58e39d9052984b7b6481cbbf50722 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Sat, 6 Sep 2025 17:44:09 +0000 Subject: [PATCH 33/57] fix: fix decode option error --- messagepack-serde/src/de/mod.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/messagepack-serde/src/de/mod.rs b/messagepack-serde/src/de/mod.rs index 13f5f3f..f2fabcf 100644 --- a/messagepack-serde/src/de/mod.rs +++ b/messagepack-serde/src/de/mod.rs @@ -304,6 +304,15 @@ mod tests { assert_eq!(decoded.schema, 0); } + #[test] + fn option_consumes_nil_in_sequence() { + // [None, 5] as an array of two elements + let buf: &[u8] = &[0x92, 0xc0, 0x05]; + + let decoded = from_slice::<(Option, u8)>(buf).unwrap(); + assert_eq!(decoded, (None, 5)); + } + #[derive(Deserialize, PartialEq, Debug)] enum E { Unit, From f34ae9347b2bdd39a33b8bde56a28651f5e9cba6 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Sat, 6 Sep 2025 19:05:25 +0000 Subject: [PATCH 34/57] doc: add doc --- messagepack-core/src/extension.rs | 2 +- messagepack-serde/src/de/error.rs | 8 ++- messagepack-serde/src/de/mod.rs | 57 +++++++----------- messagepack-serde/src/lib.rs | 14 +++-- messagepack-serde/src/ser/error.rs | 7 ++- messagepack-serde/src/ser/map.rs | 2 +- messagepack-serde/src/ser/mod.rs | 14 +++-- messagepack-serde/src/ser/num.rs | 76 ++++++++++-------------- messagepack-serde/src/ser/seq.rs | 2 +- messagepack-serde/src/value/extension.rs | 68 ++++++++++++++++++++- messagepack-serde/src/value/number.rs | 8 +++ messagepack-serde/src/value/value_.rs | 23 +++++++ 12 files changed, 183 insertions(+), 98 deletions(-) diff --git a/messagepack-core/src/extension.rs b/messagepack-core/src/extension.rs index 885ae98..0468d65 100644 --- a/messagepack-core/src/extension.rs +++ b/messagepack-core/src/extension.rs @@ -8,7 +8,7 @@ const U32_MAX: usize = u32::MAX as usize; const U8_MAX_PLUS_ONE: usize = U8_MAX + 1; const U16_MAX_PLUS_ONE: usize = U16_MAX + 1; -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct ExtensionRef<'a> { pub r#type: i8, pub data: &'a [u8], diff --git a/messagepack-serde/src/de/error.rs b/messagepack-serde/src/de/error.rs index 9e186ee..e50efad 100644 --- a/messagepack-serde/src/de/error.rs +++ b/messagepack-serde/src/de/error.rs @@ -1,14 +1,17 @@ use serde::de; -pub type CoreError = messagepack_core::decode::Error; +pub(crate) type CoreError = messagepack_core::decode::Error; +/// Error during deserialization #[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)] pub enum Error { + /// Core error Decode(CoreError), - AnyIsUnsupported, #[cfg(not(feature = "std"))] + /// Parse error Custom, #[cfg(feature = "std")] + /// Parse error Message(String), } @@ -16,7 +19,6 @@ impl core::fmt::Display for Error { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { match self { Error::Decode(e) => e.fmt(f), - Error::AnyIsUnsupported => write!(f, "Any is unsupported"), #[cfg(not(feature = "std"))] Error::Custom => write!(f, "Cannot deserialize format"), #[cfg(feature = "std")] diff --git a/messagepack-serde/src/de/mod.rs b/messagepack-serde/src/de/mod.rs index f2fabcf..f41fbee 100644 --- a/messagepack-serde/src/de/mod.rs +++ b/messagepack-serde/src/de/mod.rs @@ -1,8 +1,8 @@ mod enum_; mod error; mod seq; - -pub use error::{CoreError, Error}; +use error::CoreError; +pub use error::Error; use crate::value::extension::DeserializeExt; use messagepack_core::{Decode, Format, decode::NbyteReader}; @@ -12,8 +12,28 @@ use serde::{ forward_to_deserialize_any, }; +/// Deserialize from slice +pub fn from_slice<'de, T: Deserialize<'de>>(input: &'de [u8]) -> Result { + let mut deserializer = Deserializer::from_slice(input); + T::deserialize(&mut deserializer) +} + +#[cfg(feature = "std")] +/// Deserialize from [std::io::Read] +pub fn from_reader(reader: &mut R) -> std::io::Result +where + R: std::io::Read, + T: for<'a> Deserialize<'a>, +{ + let mut buf = Vec::new(); + reader.read_to_end(&mut buf)?; + + let mut deserializer = Deserializer::from_slice(&buf); + T::deserialize(&mut deserializer).map_err(std::io::Error::other) +} + #[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)] -pub struct Deserializer<'de> { +struct Deserializer<'de> { input: &'de [u8], } @@ -83,37 +103,6 @@ impl AsMut for Deserializer<'_> { } } -pub fn from_slice<'de, T: Deserialize<'de>>(input: &'de [u8]) -> Result { - from_slice_with_config(input) -} - -pub fn from_slice_with_config<'de, T: Deserialize<'de>>(input: &'de [u8]) -> Result { - let mut deserializer = Deserializer::from_slice(input); - T::deserialize(&mut deserializer) -} - -#[cfg(feature = "std")] -pub fn from_reader(reader: &mut R) -> std::io::Result -where - R: std::io::Read, - T: for<'a> Deserialize<'a>, -{ - from_reader_with_config(reader) -} - -#[cfg(feature = "std")] -pub fn from_reader_with_config(reader: &mut R) -> std::io::Result -where - R: std::io::Read, - T: for<'a> Deserialize<'a>, -{ - let mut buf = Vec::new(); - reader.read_to_end(&mut buf)?; - - let mut deserializer = Deserializer::from_slice(&buf); - T::deserialize(&mut deserializer).map_err(std::io::Error::other) -} - impl<'de> de::Deserializer<'de> for &mut Deserializer<'de> { type Error = Error; diff --git a/messagepack-serde/src/lib.rs b/messagepack-serde/src/lib.rs index 5869c90..acc4d6c 100644 --- a/messagepack-serde/src/lib.rs +++ b/messagepack-serde/src/lib.rs @@ -2,22 +2,28 @@ #![cfg_attr(all(not(test), not(feature = "std")), no_std)] #![cfg_attr(docsrs, feature(doc_auto_cfg))] #![doc = include_str!("../README.md")] +#![deny(missing_docs)] #[cfg(feature = "alloc")] extern crate alloc; +/// Deserialize support for messagepack pub mod de; +/// Serialize support for messagepack pub mod ser; mod value; -pub use de::{Deserializer, from_slice, from_slice_with_config}; -pub use ser::{Serializer, to_slice, to_slice_with_config}; +pub use de::from_slice; +pub use ser::{to_slice, to_slice_with_config}; #[cfg(feature = "std")] -pub use de::{from_reader, from_reader_with_config}; +pub use de::from_reader; + +#[cfg(feature = "alloc")] +pub use ser::to_vec; #[cfg(feature = "std")] -pub use ser::{to_vec, to_vec_with_config, to_writer, to_writer_with_config}; +pub use ser::{to_writer, to_writer_with_config}; pub use value::Number; #[cfg(feature = "alloc")] diff --git a/messagepack-serde/src/ser/error.rs b/messagepack-serde/src/ser/error.rs index 8ae53b8..69758da 100644 --- a/messagepack-serde/src/ser/error.rs +++ b/messagepack-serde/src/ser/error.rs @@ -1,14 +1,19 @@ use serde::ser; -pub type CoreError = messagepack_core::encode::Error; +pub(crate) type CoreError = messagepack_core::encode::Error; +/// Error during serialization #[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)] pub enum Error { + /// Core error Encode(CoreError), + /// Try serialize array or map but not passed length SeqLenNone, #[cfg(not(feature = "std"))] + /// Parse error Custom, #[cfg(feature = "std")] + /// Parse error Message(String), } diff --git a/messagepack-serde/src/ser/map.rs b/messagepack-serde/src/ser/map.rs index 194d3d2..87d2978 100644 --- a/messagepack-serde/src/ser/map.rs +++ b/messagepack-serde/src/ser/map.rs @@ -9,7 +9,7 @@ pub struct SerializeMap<'a, 'b, W, Num> { } impl<'a, 'b, W, Num> SerializeMap<'a, 'b, W, Num> { - pub(crate) fn new(ser: &'a mut Serializer<'b, W, Num>) -> Self { + pub(super) fn new(ser: &'a mut Serializer<'b, W, Num>) -> Self { Self { ser } } } diff --git a/messagepack-serde/src/ser/mod.rs b/messagepack-serde/src/ser/mod.rs index 0714f0c..c7c7f05 100644 --- a/messagepack-serde/src/ser/mod.rs +++ b/messagepack-serde/src/ser/mod.rs @@ -7,7 +7,7 @@ pub use num::{AggressiveMinimize, Exact, LosslessMinimize, NumEncoder}; use core::marker::PhantomData; use crate::value::extension::{EXTENSION_STRUCT_NAME, SerializeExt}; -pub use error::{CoreError, Error}; +pub use error::Error; use messagepack_core::{ Encode, SliceWriter, encode::{BinaryEncoder, MapFormatEncoder, NilEncoder, array::ArrayFormatEncoder}, @@ -17,7 +17,7 @@ use messagepack_core::{ use serde::ser; #[derive(Debug, PartialOrd, Ord, PartialEq, Eq)] -pub struct Serializer<'a, W, Num> { +struct Serializer<'a, W, Num> { writer: &'a mut W, current_length: usize, num_encoder: PhantomData, @@ -43,6 +43,7 @@ impl AsMut for Serializer<'_, W, Num> { } } +/// Serialize value as messagepack pub fn to_slice(value: &T, buf: &mut [u8]) -> Result> where T: ser::Serialize + ?Sized, @@ -50,6 +51,7 @@ where to_slice_with_config(value, buf, num::Exact) } +/// Serialize value as messagepack with config. pub fn to_slice_with_config<'a, T, C>( value: &T, buf: &'a mut [u8], @@ -66,6 +68,7 @@ where } #[cfg(feature = "std")] +/// Serialize value as messagepack pub fn to_writer(value: &T, writer: &mut W) -> Result> where T: ser::Serialize + ?Sized, @@ -75,6 +78,7 @@ where } #[cfg(feature = "std")] +/// Serialize value as messagepack with config. pub fn to_writer_with_config( value: &T, writer: &mut W, @@ -90,7 +94,8 @@ where Ok(ser.current_length) } -#[cfg(feature = "std")] +#[cfg(feature = "alloc")] +/// Serialize value as messagepack byte vector pub fn to_vec(value: &T) -> Result, Error> where T: ser::Serialize + ?Sized, @@ -101,7 +106,8 @@ where Ok(buf) } -#[cfg(feature = "std")] +#[cfg(feature = "alloc")] +/// Serialize value as messagepack byte vector with config pub fn to_vec_with_config(value: &T, config: C) -> Result, Error> where T: ser::Serialize + ?Sized, diff --git a/messagepack-serde/src/ser/num.rs b/messagepack-serde/src/ser/num.rs index c5254bb..51cdb30 100644 --- a/messagepack-serde/src/ser/num.rs +++ b/messagepack-serde/src/ser/num.rs @@ -7,17 +7,29 @@ use num_traits::{ToPrimitive, float::FloatCore}; /// Decide how numeric values are encoded. pub trait NumEncoder { + /// decide encode i8 fn encode_i8(v: i8, writer: &mut W) -> Result>; + /// decide encode i16 fn encode_i16(v: i16, writer: &mut W) -> Result>; + /// decide encode i32 fn encode_i32(v: i32, writer: &mut W) -> Result>; + /// decide encode i64 fn encode_i64(v: i64, writer: &mut W) -> Result>; + /// decide encode i128 fn encode_i128(v: i128, writer: &mut W) -> Result>; + /// decide encode u8 fn encode_u8(v: u8, writer: &mut W) -> Result>; + /// decide encode u16 fn encode_u16(v: u16, writer: &mut W) -> Result>; + /// decide encode u32 fn encode_u32(v: u32, writer: &mut W) -> Result>; + /// decide encode u64 fn encode_u64(v: u64, writer: &mut W) -> Result>; + /// decide encode u128 fn encode_u128(v: u128, writer: &mut W) -> Result>; + /// decide encode f32 fn encode_f32(v: f32, writer: &mut W) -> Result>; + /// decide encode f64 fn encode_f64(v: f64, writer: &mut W) -> Result>; } @@ -31,13 +43,10 @@ pub trait NumEncoder { /// /// ```rust /// use serde::Serialize; -/// use messagepack_core::SliceWriter; -/// use messagepack_serde::ser::{Serializer, Exact}; +/// use messagepack_serde::ser::{to_slice_with_config, Exact}; /// /// let mut buf = [0_u8;1]; -/// let mut writer = SliceWriter::from_slice(&mut buf); -/// let mut ser = Serializer::new(&mut writer, Exact); -/// 1_u8.serialize(&mut ser).unwrap(); +/// to_slice_with_config(&1_u8, &mut buf, Exact).unwrap(); /// /// let expected = [1_u8]; // 1 encoded in `positive fixint` /// assert_eq!(buf,expected); @@ -47,13 +56,10 @@ pub trait NumEncoder { /// /// ```rust /// use serde::Serialize; -/// use messagepack_core::SliceWriter; -/// use messagepack_serde::ser::{Serializer, Exact}; +/// use messagepack_serde::ser::{to_slice_with_config, Exact}; /// /// let mut buf = [0_u8;3]; -/// let mut writer = SliceWriter::from_slice(&mut buf); -/// let mut ser = Serializer::new(&mut writer, Exact); -/// 1_u16.serialize(&mut ser).unwrap(); +/// to_slice_with_config(&1_u16, &mut buf, Exact).unwrap(); /// /// let expected = [0xcd_u8, 0x00_u8, 1_u8]; // 1 encoded in `uint 16` /// assert_eq!(buf,expected); @@ -122,14 +128,10 @@ impl NumEncoder for Exact { /// /// ```rust /// use serde::Serialize; -/// use messagepack_core::SliceWriter; -/// use messagepack_serde::ser::{Serializer, LosslessMinimize}; +/// use messagepack_serde::ser::{to_slice_with_config, LosslessMinimize}; /// /// let mut buf = [0_u8;1]; -/// let mut writer = SliceWriter::from_slice(&mut buf); -/// let mut ser = Serializer::new(&mut writer, LosslessMinimize); -/// 1_u16.serialize(&mut ser).unwrap(); -/// +/// to_slice_with_config(&1_u16, &mut buf, LosslessMinimize).unwrap(); /// let expected = [1_u8]; // 1 encoded in `positive fixint` /// assert_eq!(buf,expected); /// ``` @@ -138,13 +140,10 @@ impl NumEncoder for Exact { /// /// ```rust /// use serde::Serialize; -/// use messagepack_core::SliceWriter; -/// use messagepack_serde::ser::{Serializer, LosslessMinimize}; +/// use messagepack_serde::ser::{to_slice_with_config, LosslessMinimize}; /// /// let mut buf = [0_u8;5]; -/// let mut writer = SliceWriter::from_slice(&mut buf); -/// let mut ser = Serializer::new(&mut writer, LosslessMinimize); -/// 1.0_f32.serialize(&mut ser).unwrap(); +/// to_slice_with_config(&1.0_f32, &mut buf, LosslessMinimize).unwrap(); /// /// let expected = [0xca,0x3f,0x80,0x00,0x00]; // 1.0 encoded in `float 32` /// assert_eq!(buf,expected); @@ -154,13 +153,10 @@ impl NumEncoder for Exact { /// /// ```rust /// use serde::Serialize; -/// use messagepack_core::SliceWriter; -/// use messagepack_serde::ser::{Serializer, LosslessMinimize}; +/// use messagepack_serde::ser::{to_slice_with_config, LosslessMinimize}; /// /// let mut buf = [0_u8;5]; -/// let mut writer = SliceWriter::from_slice(&mut buf); -/// let mut ser = Serializer::new(&mut writer, LosslessMinimize); -/// 1.0_f64.serialize(&mut ser).unwrap(); +/// to_slice_with_config(&1.0_f64, &mut buf, LosslessMinimize).unwrap(); /// /// let expected = [0xca,0x3f,0x80,0x00,0x00]; // 1.0 encoded in `float 32` /// assert_eq!(buf,expected); @@ -170,13 +166,10 @@ impl NumEncoder for Exact { /// /// ```rust /// use serde::Serialize; -/// use messagepack_core::SliceWriter; -/// use messagepack_serde::ser::{Serializer, LosslessMinimize}; +/// use messagepack_serde::ser::{to_slice_with_config, LosslessMinimize}; /// /// let mut buf = [0_u8;9]; -/// let mut writer = SliceWriter::from_slice(&mut buf); -/// let mut ser = Serializer::new(&mut writer, LosslessMinimize); -/// 0.1_f64.serialize(&mut ser).unwrap(); +/// to_slice_with_config(&0.1_f64, &mut buf, LosslessMinimize).unwrap(); /// /// let expected = [0xcb,0x3f,0xb9,0x99,0x99,0x99,0x99,0x99,0x9a]; // 0.1 encoded in `float 64` /// assert_eq!(buf,expected); @@ -262,13 +255,10 @@ impl NumEncoder for LosslessMinimize { /// /// ```rust /// use serde::Serialize; -/// use messagepack_core::SliceWriter; -/// use messagepack_serde::ser::{Serializer, AggressiveMinimize}; +/// use messagepack_serde::ser::{to_slice_with_config, AggressiveMinimize}; /// /// let mut buf = [0_u8;1]; -/// let mut writer = SliceWriter::from_slice(&mut buf); -/// let mut ser = Serializer::new(&mut writer, AggressiveMinimize); -/// 1_u16.serialize(&mut ser).unwrap(); +/// to_slice_with_config(&1_u16, &mut buf, AggressiveMinimize).unwrap(); /// /// let expected = [1_u8]; // 1 encoded in `positive fixint` /// assert_eq!(buf,expected); @@ -278,13 +268,10 @@ impl NumEncoder for LosslessMinimize { /// /// ```rust /// use serde::Serialize; -/// use messagepack_core::SliceWriter; -/// use messagepack_serde::ser::{Serializer, AggressiveMinimize}; +/// use messagepack_serde::ser::{to_slice_with_config, AggressiveMinimize}; /// /// let mut buf = [0_u8;1]; -/// let mut writer = SliceWriter::from_slice(&mut buf); -/// let mut ser = Serializer::new(&mut writer, AggressiveMinimize); -/// 1.0_f32.serialize(&mut ser).unwrap(); +/// to_slice_with_config(&1.0_f32, &mut buf, AggressiveMinimize).unwrap(); /// /// let expected = [1_u8]; // 1 encoded in `positive fixint` /// assert_eq!(buf,expected); @@ -294,13 +281,10 @@ impl NumEncoder for LosslessMinimize { /// /// ```rust /// use serde::Serialize; -/// use messagepack_core::SliceWriter; -/// use messagepack_serde::ser::{Serializer, AggressiveMinimize}; +/// use messagepack_serde::ser::{to_slice_with_config, AggressiveMinimize}; /// /// let mut buf = [0_u8;1]; -/// let mut writer = SliceWriter::from_slice(&mut buf); -/// let mut ser = Serializer::new(&mut writer, AggressiveMinimize); -/// 1.0_f64.serialize(&mut ser).unwrap(); +/// to_slice_with_config(&1.0_f64, &mut buf, AggressiveMinimize).unwrap(); /// /// let expected = [1_u8]; // 1 encoded in `positive fixint` /// assert_eq!(buf,expected); diff --git a/messagepack-serde/src/ser/seq.rs b/messagepack-serde/src/ser/seq.rs index e3c4df5..c9699e8 100644 --- a/messagepack-serde/src/ser/seq.rs +++ b/messagepack-serde/src/ser/seq.rs @@ -9,7 +9,7 @@ pub struct SerializeSeq<'a, 'b, W, Num> { } impl<'a, 'b, W, Num> SerializeSeq<'a, 'b, W, Num> { - pub(crate) fn new(ser: &'a mut Serializer<'b, W, Num>) -> Self { + pub(super) fn new(ser: &'a mut Serializer<'b, W, Num>) -> Self { Self { ser } } } diff --git a/messagepack-serde/src/value/extension.rs b/messagepack-serde/src/value/extension.rs index a7265d3..15be94b 100644 --- a/messagepack-serde/src/value/extension.rs +++ b/messagepack-serde/src/value/extension.rs @@ -5,7 +5,7 @@ use serde::{ ser::{self, SerializeSeq}, }; -use crate::ser::{CoreError, Error}; +use crate::ser::Error; pub(crate) const EXTENSION_STRUCT_NAME: &str = "$__MSGPACK_EXTENSION_STRUCT"; @@ -112,7 +112,9 @@ where } fn serialize_bytes(self, v: &[u8]) -> Result { - self.writer.write(v).map_err(CoreError::Io)?; + self.writer + .write(v) + .map_err(messagepack_core::encode::Error::Io)?; self.length += v.len(); Ok(()) } @@ -359,7 +361,9 @@ impl<'de> serde::Deserializer<'de> for &mut DeserializeExt<'de> { where V: Visitor<'de>, { - Err(crate::de::Error::AnyIsUnsupported) + Err(serde::de::Error::custom( + "any when deserialize extension is not supported", + )) } fn deserialize_i8(self, visitor: V) -> Result @@ -423,9 +427,37 @@ impl<'de> serde::de::SeqAccess<'de> for &mut DeserializeExt<'de> { } } +/// De/Serialize [messagepack_core::extension::ExtensionRef] +/// +/// ## Example +/// +/// ```rust +/// use serde::{Serialize,Deserialize}; +/// use messagepack_core::extension::ExtensionRef; +/// +/// #[derive(Debug, Serialize, Deserialize, PartialEq)] +/// #[serde(transparent)] +/// struct WrapRef<'a>( +/// #[serde(with = "messagepack_serde::ext_ref", borrow)] ExtensionRef<'a>, +/// ); +/// +/// # fn main() { +/// +/// let ext = WrapRef( +/// ExtensionRef::new(10,&[0,1,2,3,4,5]) +/// ); +/// let mut buf = [0u8; 9]; +/// messagepack_serde::to_slice(&ext, &mut buf).unwrap(); +/// +/// let result = messagepack_serde::from_slice::>(&buf).unwrap(); +/// assert_eq!(ext,result); +/// +/// # } +/// ``` pub mod ext_ref { use super::*; + /// Serialize [messagepack_core::extension::ExtensionRef] pub fn serialize( ext: &messagepack_core::extension::ExtensionRef<'_>, serializer: S, @@ -442,6 +474,7 @@ pub mod ext_ref { ) } + /// Deserialize [messagepack_core::extension::ExtensionRef] pub fn deserialize<'de, D>( deserializer: D, ) -> Result, D::Error> @@ -482,9 +515,37 @@ pub mod ext_ref { } } +/// De/Serialize [messagepack_core::extension::FixedExtension] +/// +/// ## Example +/// +/// ```rust +/// use serde::{Serialize,Deserialize}; +/// use messagepack_core::extension::FixedExtension; +/// +/// #[derive(Debug, Serialize, Deserialize, PartialEq)] +/// #[serde(transparent)] +/// struct WrapRef( +/// #[serde(with = "messagepack_serde::ext_fixed")] FixedExtension<16>, +/// ); +/// +/// # fn main() { +/// +/// let ext = WrapRef( +/// FixedExtension::new(10,&[0,1,2,3,4,5]).unwrap() +/// ); +/// let mut buf = [0u8; 9]; +/// messagepack_serde::to_slice(&ext, &mut buf).unwrap(); +/// +/// let result = messagepack_serde::from_slice::(&buf).unwrap(); +/// assert_eq!(ext,result); +/// +/// # } +/// ``` pub mod ext_fixed { use serde::de; + /// Serialize [messagepack_core::extension::FixedExtension] pub fn serialize( ext: &messagepack_core::extension::FixedExtension, serializer: S, @@ -495,6 +556,7 @@ pub mod ext_fixed { super::ext_ref::serialize(&ext.as_ref(), serializer) } + /// Deserialize [messagepack_core::extension::FixedExtension] pub fn deserialize<'de, const N: usize, D>( deserializer: D, ) -> Result, D::Error> diff --git a/messagepack-serde/src/value/number.rs b/messagepack-serde/src/value/number.rs index 91f001f..74abab7 100644 --- a/messagepack-serde/src/value/number.rs +++ b/messagepack-serde/src/value/number.rs @@ -25,24 +25,32 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Visitor}; /// ``` #[derive(Debug, Clone, Copy, PartialEq, PartialOrd)] pub enum Number { + /// Represents `positive fixint`, `uint 8`, `uint 16`, `uint 32` and `uint 64` UnsignedInt(u64), + /// Represents `negative fixint`, `int 8`, `int 16`, `int 32` and `int 64` SignedInt(i64), + /// Represents `float 32` and `float 64` Float(f64), } impl Number { + /// If the `Number` is unsigned int, returns `u64`. pub fn as_unsigned_int(&self) -> Option { match self { Number::UnsignedInt(v) => Some(*v), _ => None, } } + + /// If the `Number` is signed int, returns `i64`. pub fn as_signed_int(&self) -> Option { match self { Number::SignedInt(v) => Some(*v), _ => None, } } + + /// If the `Number` is floating number, returns `f64`. pub fn as_float(&self) -> Option { match self { Number::Float(v) => Some(*v), diff --git a/messagepack-serde/src/value/value_.rs b/messagepack-serde/src/value/value_.rs index b27be85..5126b96 100644 --- a/messagepack-serde/src/value/value_.rs +++ b/messagepack-serde/src/value/value_.rs @@ -6,56 +6,79 @@ use serde::{de::Visitor, ser::SerializeMap}; /// Represents any messagepack value. `alloc` needed. #[derive(Debug, Clone, PartialEq, PartialOrd)] pub enum ValueRef<'a> { + /// Represents nil format Nil, + /// Represents bool format family Bool(bool), + /// Represents `bin 8`, `bin 16` and `bin 32` Bin(&'a [u8]), + /// Represents ext format family Extension(ExtensionRef<'a>), + /// Represents int format family and float format family Number(Number), + /// Represents str format family String(&'a str), + /// Represents array format family Array(Vec>), + /// Represents map format family Map(Vec<(ValueRef<'a>, ValueRef<'a>)>), } impl ValueRef<'_> { + /// Returns true if the `ValueRef` is nil pub fn is_nil(&self) -> bool { matches!(self, ValueRef::Nil) } + + /// If the `ValueRef` is boolean, returns contained value. pub fn as_bool(&self) -> Option { match self { ValueRef::Bool(v) => Some(*v), _ => None, } } + + /// If the `ValueRef` is bin, returns contained value. pub fn as_bin(&self) -> Option<&[u8]> { match self { ValueRef::Bin(v) => Some(*v), _ => None, } } + + /// If the `ValueRef` is ext, returns contained value. pub fn as_extension(&self) -> Option<&ExtensionRef<'_>> { match self { ValueRef::Extension(v) => Some(v), _ => None, } } + + /// If the `ValueRef` is number, returns contained value. pub fn as_number(&self) -> Option { match self { ValueRef::Number(v) => Some(*v), _ => None, } } + + /// If the `ValueRef` is str, returns contained value. pub fn as_string(&self) -> Option<&str> { match self { ValueRef::String(v) => Some(*v), _ => None, } } + + /// If the `ValueRef` is array, returns contained value. pub fn as_array(&self) -> Option<&[ValueRef<'_>]> { match self { ValueRef::Array(v) => Some(v), _ => None, } } + + /// If the `ValueRef` is map, returns contained value. pub fn as_map(&self) -> Option<&[(ValueRef<'_>, ValueRef<'_>)]> { match self { ValueRef::Map(v) => Some(v), From 8aa45a2b6d6afecb02b86255747fc92f7b1e047f Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Sat, 6 Sep 2025 19:55:39 +0000 Subject: [PATCH 35/57] fix: deserialize unit struct --- messagepack-serde/src/de/mod.rs | 43 +++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/messagepack-serde/src/de/mod.rs b/messagepack-serde/src/de/mod.rs index f41fbee..2f58dd4 100644 --- a/messagepack-serde/src/de/mod.rs +++ b/messagepack-serde/src/de/mod.rs @@ -112,7 +112,7 @@ impl<'de> de::Deserializer<'de> for &mut Deserializer<'de> { { let format = self.decode::()?; match format { - Format::Nil => visitor.visit_none(), + Format::Nil => visitor.visit_unit(), Format::False => visitor.visit_bool(false), Format::True => visitor.visit_bool(true), Format::PositiveFixInt(v) => visitor.visit_u8(v), @@ -192,6 +192,22 @@ impl<'de> de::Deserializer<'de> for &mut Deserializer<'de> { } } + fn deserialize_option(self, visitor: V) -> Result + where + V: de::Visitor<'de>, + { + let (first, rest) = self.input.split_first().ok_or(CoreError::EofFormat)?; + + let format = Format::from_byte(*first); + match format { + Format::Nil => { + self.input = rest; + visitor.visit_none() + } + _ => visitor.visit_some(self), + } + } + fn deserialize_enum( self, _name: &'static str, @@ -227,7 +243,7 @@ impl<'de> de::Deserializer<'de> for &mut Deserializer<'de> { forward_to_deserialize_any! { bool i8 i16 i32 i64 i128 u8 u16 u32 u64 u128 f32 f64 char str string - bytes byte_buf option unit unit_struct newtype_struct seq tuple + bytes byte_buf unit unit_struct newtype_struct seq tuple tuple_struct map struct identifier ignored_any } @@ -302,6 +318,29 @@ mod tests { assert_eq!(decoded, (None, 5)); } + #[test] + fn option_some_simple() { + let buf: &[u8] = &[0x05]; + let decoded = from_slice::>(buf).unwrap(); + assert_eq!(decoded, Some(5)); + } + + #[test] + fn unit_from_nil() { + let buf: &[u8] = &[0xc0]; + let _ = from_slice::<()>(buf).unwrap(); + } + + #[test] + fn unit_struct() { + #[derive(Debug, Deserialize, PartialEq)] + struct U; + + let buf: &[u8] = &[0xc0]; + let decoded = from_slice::(buf).unwrap(); + assert_eq!(decoded, U); + } + #[derive(Deserialize, PartialEq, Debug)] enum E { Unit, From 99c1969656abf00ee23fcaa4adfda93c26978c2a Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Sat, 6 Sep 2025 20:04:59 +0000 Subject: [PATCH 36/57] fix: lint error --- messagepack-serde/src/de/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/messagepack-serde/src/de/mod.rs b/messagepack-serde/src/de/mod.rs index 2f58dd4..c66387d 100644 --- a/messagepack-serde/src/de/mod.rs +++ b/messagepack-serde/src/de/mod.rs @@ -328,7 +328,7 @@ mod tests { #[test] fn unit_from_nil() { let buf: &[u8] = &[0xc0]; - let _ = from_slice::<()>(buf).unwrap(); + from_slice::<()>(buf).unwrap(); } #[test] From 5b0720ecdd8b815ff19f24258952723d309f2072 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Sun, 7 Sep 2025 06:30:11 +0000 Subject: [PATCH 37/57] test: comp --- Cargo.lock | 1 + messagepack-serde/Cargo.toml | 2 + messagepack-serde/tests/rmp_serde.rs | 196 +++++++++++++++++++++++++++ 3 files changed, 199 insertions(+) create mode 100644 messagepack-serde/tests/rmp_serde.rs diff --git a/Cargo.lock b/Cargo.lock index 98a254f..cd064b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -311,6 +311,7 @@ version = "0.1.2" dependencies = [ "messagepack-core", "num-traits", + "rmp-serde", "rstest", "serde", "serde_bytes", diff --git a/messagepack-serde/Cargo.toml b/messagepack-serde/Cargo.toml index 6b2bb4c..cd85fc0 100644 --- a/messagepack-serde/Cargo.toml +++ b/messagepack-serde/Cargo.toml @@ -17,6 +17,8 @@ num-traits = { workspace = true } [dev-dependencies] serde_bytes = { version = "0.11" } +rmp-serde = "1.3.0" + rstest = { workspace = true } [features] diff --git a/messagepack-serde/tests/rmp_serde.rs b/messagepack-serde/tests/rmp_serde.rs new file mode 100644 index 0000000..ce5f7be --- /dev/null +++ b/messagepack-serde/tests/rmp_serde.rs @@ -0,0 +1,196 @@ +use messagepack_serde::ser::{Exact, LosslessMinimize}; +use rstest::rstest; +use serde::Serialize; +use std::collections::BTreeMap; + +fn rmp_to_vec_struct_map(value: &T) -> Vec { + let mut out = Vec::new(); + let mut ser = rmp_serde::Serializer::new(&mut out).with_struct_map(); + value.serialize(&mut ser).unwrap(); + out +} + +fn assert_same_lossless(value: &T) { + let rmp = rmp_to_vec_struct_map(value); + let mut buf = vec![0u8; rmp.len()]; + let len = messagepack_serde::to_slice_with_config(value, &mut buf, LosslessMinimize).unwrap(); + assert_eq!(rmp, buf[..len]); +} + +fn assert_same_exact(value: &T) { + let rmp = rmp_to_vec_struct_map(value); + let mut buf = vec![0u8; rmp.len()]; + let len = messagepack_serde::to_slice_with_config(value, &mut buf, Exact).unwrap(); + assert_eq!(rmp, buf[..len]); +} + +#[test] +fn bool_nil_char() { + assert_same_lossless(&true); + assert_same_lossless(&false); + // Note: rmp-serde encodes unit and unit struct as empty array (fixarray 0), + // while messagepack_serde encodes as nil. Exclude from parity checks. + assert_same_lossless(&Option::::None); + assert_same_lossless(&'a'); + assert_same_lossless(&'😀'); +} + +#[test] +fn integers_boundaries() { + // signed + assert_same_lossless(&-1i64); + assert_same_lossless(&-32i64); // neg fixint + assert_same_lossless(&-33i64); // int8 boundary + assert_same_lossless(&-128i64); + assert_same_lossless(&-129i64); + assert_same_lossless(&-32768i64); + assert_same_lossless(&-32769i64); + assert_same_lossless(&-2147483648i64); + assert_same_lossless(&-2147483649i64); + assert_same_lossless(&i64::MIN); + assert_same_lossless(&i64::MAX); + + // unsigned + assert_same_lossless(&0u64); + assert_same_lossless(&1u64); + assert_same_lossless(&127u64); // pos fixint + assert_same_lossless(&128u64); // uint8 + assert_same_lossless(&255u64); + assert_same_lossless(&256u64); // uint16 + assert_same_lossless(&65535u64); + assert_same_lossless(&65536u64); // uint32 + assert_same_lossless(&4294967295u64); + assert_same_lossless(&4294967296u64); // u64 boundary + assert_same_lossless(&u64::MAX); +} + +#[test] +fn floats_f32() { + assert_same_lossless(&0.0f32); + assert_same_lossless(&-0.0f32); + assert_same_lossless(&1.5f32); + assert_same_lossless(&f32::INFINITY); + assert_same_lossless(&f32::NEG_INFINITY); + assert_same_lossless(&f32::from_bits(0x7FC0_0000)); // canonical quiet NaN +} + +#[test] +fn floats_f64_exact() { + // rmp-serde encodes f64 as f64; use Exact to match + assert_same_exact(&0.0f64); + assert_same_exact(&-0.0f64); + assert_same_exact(&1.5f64); + assert_same_exact(&f64::INFINITY); + assert_same_exact(&f64::NEG_INFINITY); + assert_same_exact(&f64::from_bits(0x7FF8_0000_0000_0000)); // quiet NaN +} + +#[rstest] +#[case(0usize)] +#[case(31)] // fixstr upper +#[case(32)] // str8 +#[case(255)] +#[case(256)] // str16 +#[case(1024)] +fn strings_boundaries(#[case] len: usize) { + let s = "a".repeat(len); + assert_same_lossless(&s); +} + +#[rstest] +#[case(0usize)] +#[case(1)] +#[case(255)] // bin8 upper +#[case(256)] // bin16 +#[case(1024)] +fn binary_boundaries(#[case] len: usize) { + let data = vec![0u8; len]; + let bb = serde_bytes::ByteBuf::from(data); + assert_same_lossless(&bb); +} + +#[test] +fn arrays_simple() { + let v0: Vec = vec![]; + let v15: Vec = (0..15).map(|i| i as i64).collect(); + let v16: Vec = (0..16).map(|i| i as i64).collect(); + assert_same_lossless(&v0); + assert_same_lossless(&v15); + assert_same_lossless(&v16); +} + +#[test] +fn maps_btreemap_order_and_sizes() { + let m0: BTreeMap = BTreeMap::new(); + assert_same_lossless(&m0); + + let mut m1 = BTreeMap::new(); + m1.insert(1, 10); + assert_same_lossless(&m1); + + let mut m15 = BTreeMap::new(); + for i in 0..15u32 { m15.insert(i, i+100); } + assert_same_lossless(&m15); + + let mut m16 = BTreeMap::new(); + for i in 0..16u32 { m16.insert(i, i+100); } + assert_same_lossless(&m16); +} + +#[test] +fn structs_as_maps() { + #[derive(Serialize)] + struct Named { + a: u8, + b: Option<&'static str>, + } + #[derive(Serialize)] + struct Tuple(u8, i16); + #[derive(Serialize)] + struct WithRename { + #[serde(rename = "id")] + ident: u32, + #[serde(rename = "msg")] + message: &'static str, + } + + assert_same_lossless(&Named { a: 7, b: Some("hi") }); + assert_same_lossless(&Named { a: 7, b: None }); + assert_same_lossless(&Tuple(1, -2)); + assert_same_lossless(&WithRename { ident: 1, message: "ok" }); +} + +#[test] +fn enums_external_and_tagged() { + #[derive(Serialize)] + enum E { + A, + B(i32), + C(i32, i32), + D { x: u8, y: bool }, + } + assert_same_lossless(&E::A); + assert_same_lossless(&E::B(10)); + assert_same_lossless(&E::C(1, 2)); + assert_same_lossless(&E::D { x: 3, y: true }); + + #[derive(Serialize)] + #[serde(tag = "t", content = "c")] + enum Adj { + U, + V(u8), + W { k: &'static str }, + } + assert_same_lossless(&Adj::U); + assert_same_lossless(&Adj::V(5)); + assert_same_lossless(&Adj::W { k: "v" }); + + #[derive(Serialize)] + #[serde(tag = "type")] + enum Internal { + A, + B { x: i64 }, + } + assert_same_lossless(&Internal::A); + assert_same_lossless(&Internal::B { x: 42 }); +} From 8be8d10ac79a7bb1ff6a5be2962b34e8e250baa9 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Sun, 7 Sep 2025 06:45:04 +0000 Subject: [PATCH 38/57] perf: add size_hint --- messagepack-serde/src/de/seq.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/messagepack-serde/src/de/seq.rs b/messagepack-serde/src/de/seq.rs index 270495b..9313068 100644 --- a/messagepack-serde/src/de/seq.rs +++ b/messagepack-serde/src/de/seq.rs @@ -33,6 +33,10 @@ where Ok(None) } } + + fn size_hint(&self) -> Option { + Some(self.left) + } } impl<'de, 'a> de::MapAccess<'de> for FixLenAccess<'de, 'a> @@ -62,4 +66,8 @@ where { seed.deserialize(self.de.as_mut()) } + + fn size_hint(&self) -> Option { + Some(self.left) + } } From 34cef8cc99b37b1aac95490b3087d8f22c51f44f Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Sun, 7 Sep 2025 06:47:23 +0000 Subject: [PATCH 39/57] chore: cargo fmt --- messagepack-serde/src/de/seq.rs | 2 +- messagepack-serde/tests/rmp_serde.rs | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/messagepack-serde/src/de/seq.rs b/messagepack-serde/src/de/seq.rs index 9313068..ccae5ce 100644 --- a/messagepack-serde/src/de/seq.rs +++ b/messagepack-serde/src/de/seq.rs @@ -33,7 +33,7 @@ where Ok(None) } } - + fn size_hint(&self) -> Option { Some(self.left) } diff --git a/messagepack-serde/tests/rmp_serde.rs b/messagepack-serde/tests/rmp_serde.rs index ce5f7be..9f30c59 100644 --- a/messagepack-serde/tests/rmp_serde.rs +++ b/messagepack-serde/tests/rmp_serde.rs @@ -129,11 +129,15 @@ fn maps_btreemap_order_and_sizes() { assert_same_lossless(&m1); let mut m15 = BTreeMap::new(); - for i in 0..15u32 { m15.insert(i, i+100); } + for i in 0..15u32 { + m15.insert(i, i + 100); + } assert_same_lossless(&m15); let mut m16 = BTreeMap::new(); - for i in 0..16u32 { m16.insert(i, i+100); } + for i in 0..16u32 { + m16.insert(i, i + 100); + } assert_same_lossless(&m16); } @@ -154,10 +158,16 @@ fn structs_as_maps() { message: &'static str, } - assert_same_lossless(&Named { a: 7, b: Some("hi") }); + assert_same_lossless(&Named { + a: 7, + b: Some("hi"), + }); assert_same_lossless(&Named { a: 7, b: None }); assert_same_lossless(&Tuple(1, -2)); - assert_same_lossless(&WithRename { ident: 1, message: "ok" }); + assert_same_lossless(&WithRename { + ident: 1, + message: "ok", + }); } #[test] From 973aee8ba0636a3e24f4673c279f16b4412c93a6 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Sun, 7 Sep 2025 11:31:26 +0000 Subject: [PATCH 40/57] fix: check decode with format --- messagepack-core/src/decode/nil.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/messagepack-core/src/decode/nil.rs b/messagepack-core/src/decode/nil.rs index 0de764e..5e40430 100644 --- a/messagepack-core/src/decode/nil.rs +++ b/messagepack-core/src/decode/nil.rs @@ -8,16 +8,15 @@ impl<'a> Decode<'a> for NilDecoder { fn decode(buf: &'a [u8]) -> Result<(Self::Value, &'a [u8])> { let (format, buf) = Format::decode(buf)?; + Self::decode_with_format(format, buf) + } + + fn decode_with_format(format: Format, buf: &'a [u8]) -> Result<(Self::Value, &'a [u8])> { match format { Format::Nil => Ok(((), buf)), _ => Err(Error::UnexpectedFormat), } } - - fn decode_with_format(format: Format, buf: &'a [u8]) -> Result<(Self::Value, &'a [u8])> { - let _ = format; - Ok(((), buf)) - } } impl<'a> Decode<'a> for () { From f807d2c6cfd95ca181b6c2e2510487ff61a22cd3 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Sun, 7 Sep 2025 12:40:19 +0000 Subject: [PATCH 41/57] doc: add doc --- messagepack-core/src/decode/array.rs | 3 ++ messagepack-core/src/decode/bin.rs | 3 ++ messagepack-core/src/decode/map.rs | 3 ++ messagepack-core/src/decode/mod.rs | 20 +++++++++++-- messagepack-core/src/decode/nil.rs | 3 ++ messagepack-core/src/decode/str.rs | 3 ++ messagepack-core/src/encode/array.rs | 3 ++ messagepack-core/src/encode/bin.rs | 3 ++ messagepack-core/src/encode/bool.rs | 2 ++ messagepack-core/src/encode/float.rs | 4 +++ messagepack-core/src/encode/int.rs | 2 ++ messagepack-core/src/encode/map.rs | 12 ++++++++ messagepack-core/src/encode/mod.rs | 16 ++++++++--- messagepack-core/src/encode/nil.rs | 3 ++ messagepack-core/src/encode/str.rs | 5 ++++ messagepack-core/src/extension.rs | 34 ++++++++++++++++++++++ messagepack-core/src/formats.rs | 43 ++++++++++++++++++++++++++++ messagepack-core/src/io.rs | 8 ++++++ messagepack-core/src/lib.rs | 1 + messagepack-core/src/timestamp.rs | 12 ++++++++ 20 files changed, 176 insertions(+), 7 deletions(-) diff --git a/messagepack-core/src/decode/array.rs b/messagepack-core/src/decode/array.rs index 597184c..b3cf9f7 100644 --- a/messagepack-core/src/decode/array.rs +++ b/messagepack-core/src/decode/array.rs @@ -1,8 +1,11 @@ +//! Array decoding helpers. + use core::marker::PhantomData; use super::{Decode, Error, NbyteReader, Result}; use crate::formats::Format; +/// Decode a MessagePack array of `V` into `Array` collecting iterator. pub struct ArrayDecoder(PhantomData<(Array, V)>); impl<'a, Array, V> Decode<'a> for ArrayDecoder diff --git a/messagepack-core/src/decode/bin.rs b/messagepack-core/src/decode/bin.rs index 7c4691b..cafe7de 100644 --- a/messagepack-core/src/decode/bin.rs +++ b/messagepack-core/src/decode/bin.rs @@ -1,6 +1,9 @@ +//! Binary (bin8/16/32) decoding helpers. + use super::{Decode, Error, NbyteReader, Result}; use crate::formats::Format; +/// Decode a MessagePack binary blob and return a borrowed byte slice. pub struct BinDecoder; impl<'a> Decode<'a> for BinDecoder { diff --git a/messagepack-core/src/decode/map.rs b/messagepack-core/src/decode/map.rs index 3107dab..4f251d6 100644 --- a/messagepack-core/src/decode/map.rs +++ b/messagepack-core/src/decode/map.rs @@ -1,8 +1,11 @@ +//! Map decoding helpers. + use core::marker::PhantomData; use super::{Decode, Error, NbyteReader, Result}; use crate::formats::Format; +/// Decode a MessagePack map of `K -> V` into `Map` collecting iterator. pub struct MapDecoder(PhantomData<(Map, K, V)>); fn decode_kv<'a, K, V>(buf: &'a [u8]) -> Result<(K::Value, V::Value, &'a [u8])> diff --git a/messagepack-core/src/decode/mod.rs b/messagepack-core/src/decode/mod.rs index d6ddb25..c9c820b 100644 --- a/messagepack-core/src/decode/mod.rs +++ b/messagepack-core/src/decode/mod.rs @@ -1,3 +1,8 @@ +//! Decoding primitives for MessagePack. +//! +//! This module provides low-level decoders that operate on byte slices and +//! return decoded values along with the remaining tail. + use crate::Format; mod array; @@ -15,7 +20,7 @@ mod str; pub use str::StrDecoder; mod timestamp; -/// Messagepack Encode Error +/// MessagePack decode error #[derive(Debug, Copy, Clone, PartialOrd, Ord, PartialEq, Eq)] pub enum Error { /// Invalid data @@ -41,14 +46,20 @@ impl core::fmt::Display for Error { impl core::error::Error for Error {} +/// Result alias used by the decoders in this module. type Result = ::core::result::Result; +/// A type that can be decoded from a MessagePack byte slice. pub trait Decode<'a> { + /// The materialised value type. type Value: Sized; - // decode from buf and return (value,rest) + /// Decode a value from the front of `buf`, returning the value and the + /// remaining slice. fn decode(buf: &'a [u8]) -> Result<(Self::Value, &'a [u8])>; - // decode with format + /// Decode a value assuming the leading MessagePack format has already been + /// read by the caller. Implementations must validate that `format` is + /// appropriate for the type and return an error otherwise. fn decode_with_format(format: Format, buf: &'a [u8]) -> Result<(Self::Value, &'a [u8])>; } @@ -66,10 +77,13 @@ impl<'a> Decode<'a> for Format { } } +/// Helper to read a fixed number of big‑endian bytes and return them as `usize`. pub struct NbyteReader; macro_rules! impl_read { ($ty:ty) => { + /// Read the next big‑endian integer of type `$ty` and return it as + /// `usize` together with the remaining slice. pub fn read(buf: &[u8]) -> Result<(usize, &[u8])> { const SIZE: usize = core::mem::size_of::<$ty>(); let (data, rest) = buf.split_at_checked(SIZE).ok_or(Error::EofData)?; diff --git a/messagepack-core/src/decode/nil.rs b/messagepack-core/src/decode/nil.rs index 5e40430..bea626b 100644 --- a/messagepack-core/src/decode/nil.rs +++ b/messagepack-core/src/decode/nil.rs @@ -1,6 +1,9 @@ +//! Nil and `Option` decoding helpers. + use super::{Decode, Error, Result}; use crate::formats::Format; +/// Decode the MessagePack `nil` value. pub struct NilDecoder; impl<'a> Decode<'a> for NilDecoder { diff --git a/messagepack-core/src/decode/str.rs b/messagepack-core/src/decode/str.rs index 9f5ec07..438abed 100644 --- a/messagepack-core/src/decode/str.rs +++ b/messagepack-core/src/decode/str.rs @@ -1,6 +1,9 @@ +//! String decoding helpers. + use super::{Decode, Error, NbyteReader, Result}; use crate::formats::Format; +/// Decode a MessagePack string and return a borrowed `&str`. pub struct StrDecoder; impl<'a> Decode<'a> for StrDecoder { diff --git a/messagepack-core/src/encode/array.rs b/messagepack-core/src/encode/array.rs index 5b3dc9e..cc3cb64 100644 --- a/messagepack-core/src/encode/array.rs +++ b/messagepack-core/src/encode/array.rs @@ -1,6 +1,9 @@ +//! Array format encoder. + use super::{Encode, Error, Result}; use crate::{formats::Format, io::IoWrite}; +/// Encode only the array header for an array of a given length. pub struct ArrayFormatEncoder(pub usize); impl Encode for ArrayFormatEncoder { diff --git a/messagepack-core/src/encode/bin.rs b/messagepack-core/src/encode/bin.rs index 806093f..424c3ec 100644 --- a/messagepack-core/src/encode/bin.rs +++ b/messagepack-core/src/encode/bin.rs @@ -1,6 +1,9 @@ +//! Binary encoders. + use super::{Encode, Error, Result}; use crate::{formats::Format, io::IoWrite}; +/// Encoder for MessagePack binary values (`bin8/16/32`). #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] pub struct BinaryEncoder<'blob>(pub &'blob [u8]); diff --git a/messagepack-core/src/encode/bool.rs b/messagepack-core/src/encode/bool.rs index 986d9c2..84fbfcb 100644 --- a/messagepack-core/src/encode/bool.rs +++ b/messagepack-core/src/encode/bool.rs @@ -1,3 +1,5 @@ +//! Boolean encoders. + use super::{Encode, Result}; use crate::{formats::Format, io::IoWrite}; diff --git a/messagepack-core/src/encode/float.rs b/messagepack-core/src/encode/float.rs index ad52ad3..edcf3ca 100644 --- a/messagepack-core/src/encode/float.rs +++ b/messagepack-core/src/encode/float.rs @@ -1,3 +1,5 @@ +//! Floating‑point encoders. + use super::{Encode, Result}; use crate::{formats::Format, io::IoWrite}; @@ -30,7 +32,9 @@ fn is_exactly_representable(x: f64) -> bool { /// encode minimum byte size #[derive(Debug, Copy, Clone, PartialOrd, PartialEq)] pub enum EncodeMinimizeFloat { + /// Encode as `float32` if exact, otherwise upcast to `float64`. F32(f32), + /// Always encode as `float64`. F64(f64), } diff --git a/messagepack-core/src/encode/int.rs b/messagepack-core/src/encode/int.rs index 0ba52c9..42a6032 100644 --- a/messagepack-core/src/encode/int.rs +++ b/messagepack-core/src/encode/int.rs @@ -1,3 +1,5 @@ +//! Integer encoders and size‑minimising helpers. + use num_traits::ToPrimitive; use super::{Encode, Error, Result}; diff --git a/messagepack-core/src/encode/map.rs b/messagepack-core/src/encode/map.rs index 64c90b2..8f18005 100644 --- a/messagepack-core/src/encode/map.rs +++ b/messagepack-core/src/encode/map.rs @@ -1,12 +1,16 @@ +//! Map encoders. + use core::{cell::RefCell, marker::PhantomData, ops::Deref}; use super::{Encode, Error, Result}; use crate::{formats::Format, io::IoWrite}; +/// A key-value encoder that writes a single `key, value` pair. pub trait KVEncode where W: IoWrite, { + /// Encode this key‑value pair to the writer and return the number of bytes written. fn encode(&self, writer: &mut W) -> Result; } @@ -25,8 +29,10 @@ impl, V: Encode> KVEncode for (K, V) { } } +/// Encode only the map header for a map of a given length. pub struct MapFormatEncoder(pub usize); impl MapFormatEncoder { + /// Construct from the number of pairs contained in the map. pub fn new(size: usize) -> Self { Self(size) } @@ -58,6 +64,7 @@ impl Encode for MapFormatEncoder { } } +/// Encode a stream of key-value pairs from an iterator. pub struct MapDataEncoder { data: RefCell, _phantom: PhantomData<(I, J, KV)>, @@ -67,6 +74,7 @@ impl MapDataEncoder where I: IntoIterator, { + /// Construct from any iterable of key-value pairs. pub fn new(data: I) -> Self { Self { data: RefCell::new(data.into_iter()), @@ -92,12 +100,14 @@ where } } +/// Encode a slice of key-value pairs. pub struct MapSliceEncoder<'data, KV> { data: &'data [KV], _phantom: PhantomData, } impl<'data, KV> MapSliceEncoder<'data, KV> { + /// Construct from a slice of key-value pairs. pub fn new(data: &'data [KV]) -> Self { Self { data, @@ -127,6 +137,7 @@ where } } +/// Encode a map from an owned iterator, writing items lazily. pub struct MapEncoder { map: RefCell, _phantom: PhantomData<(W, I, J, KV)>, @@ -138,6 +149,7 @@ where I: IntoIterator, KV: KVEncode, { + /// Construct from any iterable of key-value pairs. pub fn new(map: I) -> Self { Self { map: RefCell::new(map.into_iter()), diff --git a/messagepack-core/src/encode/mod.rs b/messagepack-core/src/encode/mod.rs index f3c09ef..8ba6497 100644 --- a/messagepack-core/src/encode/mod.rs +++ b/messagepack-core/src/encode/mod.rs @@ -1,3 +1,8 @@ +//! Encoding primitives for MessagePack. +//! +//! This module exposes the `Encode` trait and a number of small helper +//! encoders for arrays, maps, strings and binary data. + pub mod array; pub mod bin; pub mod bool; @@ -8,16 +13,19 @@ pub mod nil; pub mod str; mod timestamp; +/// Helper to encode raw binary blobs using `bin8/16/32` formats. pub use bin::BinaryEncoder; +/// Helpers to encode MessagePack maps from various sources. pub use map::{MapDataEncoder, MapEncoder, MapFormatEncoder, MapSliceEncoder}; +/// Encode the MessagePack `nil` value. pub use nil::NilEncoder; use crate::{Format, io::IoWrite}; -/// Messagepack Encode Error +/// MessagePack encode error #[derive(Debug, Copy, Clone, PartialOrd, Ord, PartialEq, Eq)] pub enum Error { - // io error + /// Error produced by the underlying writer. Io(T), /// Cannot mapped messagepack format InvalidFormat, @@ -42,12 +50,12 @@ impl core::error::Error for Error {} type Result = ::core::result::Result>; -/// A type which can be encoded to MessagePack +/// A type which can be encoded to MessagePack. pub trait Encode where W: IoWrite, { - /// encode to MessagePack + /// Encode this value to MessagePack and write bytes to `writer`. fn encode(&self, writer: &mut W) -> Result; } diff --git a/messagepack-core/src/encode/nil.rs b/messagepack-core/src/encode/nil.rs index a3c2004..fae2dc1 100644 --- a/messagepack-core/src/encode/nil.rs +++ b/messagepack-core/src/encode/nil.rs @@ -1,6 +1,9 @@ +//! Nil encoder. + use super::{Encode, Result}; use crate::{formats::Format, io::IoWrite}; +/// Encode the MessagePack `nil` value. pub struct NilEncoder; impl Encode for NilEncoder { diff --git a/messagepack-core/src/encode/str.rs b/messagepack-core/src/encode/str.rs index d7bdb66..369a0c0 100644 --- a/messagepack-core/src/encode/str.rs +++ b/messagepack-core/src/encode/str.rs @@ -1,8 +1,11 @@ +//! String encoders. + use core::ops::Deref; use super::{Encode, Error, Result}; use crate::{formats::Format, io::IoWrite}; +/// Encode only the string header for a string of a given byte length. pub struct StrFormatEncoder(pub usize); impl Encode for StrFormatEncoder { fn encode(&self, writer: &mut W) -> Result::Error> { @@ -35,6 +38,7 @@ impl Encode for StrFormatEncoder { } } +/// Encode only the string bytes without a header. pub struct StrDataEncoder<'a>(pub &'a str); impl Encode for StrDataEncoder<'_> { fn encode(&self, writer: &mut W) -> Result::Error> { @@ -43,6 +47,7 @@ impl Encode for StrDataEncoder<'_> { Ok(self.0.len()) } } +/// Encode a `&str` including its appropriate header. pub struct StrEncoder<'s>(pub &'s str); impl<'s> Deref for StrEncoder<'s> { diff --git a/messagepack-core/src/extension.rs b/messagepack-core/src/extension.rs index 0468d65..07ac45f 100644 --- a/messagepack-core/src/extension.rs +++ b/messagepack-core/src/extension.rs @@ -1,3 +1,5 @@ +//! MessagePack extension helpers. + use crate::decode::{self, NbyteReader}; use crate::encode; use crate::{Decode, Encode, formats::Format, io::IoWrite}; @@ -8,17 +10,28 @@ const U32_MAX: usize = u32::MAX as usize; const U8_MAX_PLUS_ONE: usize = U8_MAX + 1; const U16_MAX_PLUS_ONE: usize = U16_MAX + 1; +/// A borrowed view of a MessagePack extension value. +/// +/// Note that the MessagePack header (FixExt vs Ext8/16/32) is determined by the +/// payload length when encoding. See [`ExtensionRef::to_format`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct ExtensionRef<'a> { + /// Application‑defined extension type code. pub r#type: i8, + /// Borrowed payload bytes. pub data: &'a [u8], } impl<'a> ExtensionRef<'a> { + /// Create a borrowed reference to extension data with the given type code. pub fn new(r#type: i8, data: &'a [u8]) -> Self { Self { r#type, data } } + /// Decide the MessagePack format to use given the payload length. + /// + /// - If `data.len()` is exactly 1, 2, 4, 8 or 16, `FixExtN` is selected. + /// - Otherwise, `Ext8`/`Ext16`/`Ext32` is selected based on the byte length. pub fn to_format(&self) -> core::result::Result> { let format = match self.data.len() { 1 => Format::FixExt1, @@ -139,14 +152,26 @@ impl<'a> Decode<'a> for ExtensionRef<'a> { } } +/// A fixed-capacity container for extension payloads of up to `N` bytes. +/// +/// This type name refers to the fixed-size backing buffer, not the MessagePack +/// header kind. The actual header used at encode-time depends on the current +/// payload length: +/// - `len == 1, 2, 4, 8, 16` → `FixExtN` +/// - otherwise (0..=255, 256..=65535, 65536..=u32::MAX) → `Ext8/16/32` #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] pub struct FixedExtension { + /// Application‑defined extension type code. pub r#type: i8, len: usize, data: [u8; N], } impl FixedExtension { + /// Construct from a slice whose length must be `<= N`. + /// + /// The chosen MessagePack format when encoding still follows the rules + /// described in the type-level documentation above. pub fn new(r#type: i8, data: &[u8]) -> Option { if data.len() > N { return None; @@ -160,6 +185,11 @@ impl FixedExtension { }) } + /// Construct with an exact `N`-byte payload. + /// + /// Note: Even when constructed with a fixed-size buffer, the encoder will + /// emit `FixExtN` only if `N` is one of {1, 2, 4, 8, 16}. For any other + /// `N`, the encoder uses `Ext8/16/32` as appropriate. pub fn new_fixed(r#type: i8, data: [u8; N]) -> Self { Self { r#type, @@ -168,6 +198,7 @@ impl FixedExtension { } } + /// Borrow as [`ExtensionRef`] for encoding. pub fn as_ref(&self) -> ExtensionRef<'_> { ExtensionRef { r#type: self.r#type, @@ -175,14 +206,17 @@ impl FixedExtension { } } + /// Current payload length in bytes. pub fn len(&self) -> usize { self.len } + /// Returns `true` if the payload is empty. pub fn is_empty(&self) -> bool { self.len == 0 } + /// Borrow the payload slice. pub fn data(&self) -> &[u8] { &self.data[..self.len] } diff --git a/messagepack-core/src/formats.rs b/messagepack-core/src/formats.rs index bc3765f..a84b190 100644 --- a/messagepack-core/src/formats.rs +++ b/messagepack-core/src/formats.rs @@ -1,3 +1,5 @@ +//! MessagePack format markers. +//! //! See const POSITIVE_FIXINT: u8 = 0x00; @@ -40,48 +42,87 @@ const ARRAY32: u8 = 0xdd; const MAP16: u8 = 0xde; const MAP32: u8 = 0xdf; +/// MessagePack format marker. #[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)] pub enum Format { + /// Positive fixint (0xxxxxxx): stores a positive 7‑bit integer in the marker. PositiveFixInt(u8), + /// Fixmap (1000xxxx): small map with length encoded in the marker. FixMap(u8), + /// Fixarray (1001xxxx): small array with length encoded in the marker. FixArray(u8), + /// Fixstr (101xxxxx): small string with byte length in the marker. FixStr(u8), + /// Nil (0xc0). Nil, + /// Reserved (0xc1): never used. NeverUsed, + /// False (0xc2). False, + /// True (0xc3). True, + /// Binary with 8‑bit length (0xc4). Bin8, + /// Binary with 16‑bit length (0xc5). Bin16, + /// Binary with 32‑bit length (0xc6). Bin32, + /// Extension with 8‑bit length (0xc7). Ext8, + /// Extension with 16‑bit length (0xc8). Ext16, + /// Extension with 32‑bit length (0xc9). Ext32, + /// Float32 (0xca). Float32, + /// Float64 (0xcb). Float64, + /// Unsigned 8‑bit integer (0xcc). Uint8, + /// Unsigned 16‑bit integer (0xcd). Uint16, + /// Unsigned 32‑bit integer (0xce). Uint32, + /// Unsigned 64‑bit integer (0xcf). Uint64, + /// Signed 8‑bit integer (0xd0). Int8, + /// Signed 16‑bit integer (0xd1). Int16, + /// Signed 32‑bit integer (0xd2). Int32, + /// Signed 64‑bit integer (0xd3). Int64, + /// Fixext 1 (0xd4). FixExt1, + /// Fixext 2 (0xd5). FixExt2, + /// Fixext 4 (0xd6). FixExt4, + /// Fixext 8 (0xd7). FixExt8, + /// Fixext 16 (0xd8). FixExt16, + /// Str8: UTF‑8 string with 8‑bit length (0xd9). Str8, + /// Str16: UTF‑8 string with 16‑bit length (0xda). Str16, + /// Str32: UTF‑8 string with 32‑bit length (0xdb). Str32, + /// Array16: array with 16‑bit length (0xdc). Array16, + /// Array32: array with 32‑bit length (0xdd). Array32, + /// Map16: map with 16‑bit length (0xde). Map16, + /// Map32: map with 32‑bit length (0xdf). Map32, + /// Negative fixint (111xxxxx): stores a negative 5‑bit integer in the marker. NegativeFixInt(i8), } impl Format { + /// Return the marker byte for this format. pub const fn as_byte(&self) -> u8 { match self { Format::PositiveFixInt(v) => POSITIVE_FIXINT | *v, @@ -124,6 +165,7 @@ impl Format { } } + /// Parse a marker byte into a [`Format`] value. pub const fn from_byte(byte: u8) -> Self { match byte { 0x00..=0x7f => Self::PositiveFixInt(byte & !POSITIVE_FIXINT), @@ -166,6 +208,7 @@ impl Format { } } + /// Return the marker byte wrapped in a single‑byte array. pub const fn as_slice(&self) -> [u8; 1] { self.as_byte().to_be_bytes() } diff --git a/messagepack-core/src/io.rs b/messagepack-core/src/io.rs index 99be755..8704fe3 100644 --- a/messagepack-core/src/io.rs +++ b/messagepack-core/src/io.rs @@ -1,5 +1,11 @@ +//! Minimal write abstraction used by encoders. + +/// Minimal `Write`‑like trait used by encoders to avoid committing to a +/// specific I/O model. pub trait IoWrite { + /// Error type produced by the writer. type Error: core::error::Error; + /// Write all bytes from `buf`. fn write(&mut self, buf: &[u8]) -> Result<(), Self::Error>; } @@ -20,12 +26,14 @@ impl core::fmt::Display for WError { impl core::error::Error for WError {} +/// Simple writer that writes into a mutable byte slice. pub struct SliceWriter<'a> { buf: &'a mut [u8], cursor: usize, } impl<'a> SliceWriter<'a> { + /// Create a new writer over the given buffer. pub fn from_slice(buf: &'a mut [u8]) -> Self { Self { buf, cursor: 0 } } diff --git a/messagepack-core/src/lib.rs b/messagepack-core/src/lib.rs index 02cf5f8..a7349de 100644 --- a/messagepack-core/src/lib.rs +++ b/messagepack-core/src/lib.rs @@ -2,6 +2,7 @@ #![cfg_attr(all(not(test), not(feature = "std")), no_std)] #![cfg_attr(docsrs, feature(doc_auto_cfg))] #![doc = include_str!("../README.md")] +#![deny(missing_docs)] pub mod decode; pub mod encode; diff --git a/messagepack-core/src/timestamp.rs b/messagepack-core/src/timestamp.rs index 86021e7..36f0787 100644 --- a/messagepack-core/src/timestamp.rs +++ b/messagepack-core/src/timestamp.rs @@ -1,3 +1,5 @@ +//! MessagePack timestamp extension values. + pub(crate) const TIMESTAMP_EXTENSION_TYPE: i8 = -1; /// Represents timestamp 32 extension type. @@ -8,10 +10,12 @@ pub struct Timestamp32 { } impl Timestamp32 { + /// Create a 32‑bit seconds timestamp. pub fn new(seconds: u32) -> Self { Self { secs: seconds } } + /// Get seconds since the UNIX epoch. pub fn seconds(&self) -> u32 { self.secs } @@ -37,11 +41,14 @@ pub struct Timestamp64 { /// `seconds` or `nanos` cannot be represented #[derive(Clone, Debug)] pub struct Timestamp64Error { + /// Requested seconds that exceeded the 34‑bit range. pub seconds: u64, + /// Requested nanoseconds that exceeded the 30‑bit range. pub nanos: u32, } impl Timestamp64 { + /// Create a 64‑bit timestamp storing 34‑bit seconds and 30‑bit nanoseconds. pub fn new(seconds: u64, nanos: u32) -> Result { const SECONDS_MAX_LIMIT: u64 = 1 << 34; @@ -64,6 +71,7 @@ impl Timestamp64 { Ok(Self::from_buf(buf)) } + /// Get the nanoseconds component. pub fn nanos(&self) -> u32 { let mut buf = [0u8; 4]; buf.copy_from_slice(&self.data[..4]); @@ -71,6 +79,7 @@ impl Timestamp64 { nanosec >> 2 } + /// Get the seconds component. pub fn seconds(&self) -> u64 { // 34bit mask const MASK: u64 = (1 << 34) - 1; @@ -99,6 +108,7 @@ pub struct Timestamp96 { } impl Timestamp96 { + /// Create a 96‑bit timestamp storing signed seconds and nanoseconds. pub fn new(seconds: i64, nanoseconds: u32) -> Self { Self { nanos: nanoseconds, @@ -106,10 +116,12 @@ impl Timestamp96 { } } + /// Get the nanoseconds component. pub fn nanos(&self) -> u32 { self.nanos } + /// Get the seconds component. pub fn seconds(&self) -> i64 { self.secs } From 95b342ad8e7c269697f22e1dd7bc3225cdbe04dc Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Sun, 7 Sep 2025 13:09:27 +0000 Subject: [PATCH 42/57] test: serialize struct as map --- messagepack-bench/benches/serialization.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/messagepack-bench/benches/serialization.rs b/messagepack-bench/benches/serialization.rs index bcd888b..25c3cf2 100644 --- a/messagepack-bench/benches/serialization.rs +++ b/messagepack-bench/benches/serialization.rs @@ -58,7 +58,7 @@ fn serialize_rmp_serde(bencher: divan::Bencher, l bencher.bench_local_refs(|buf| { let buf = core::hint::black_box(buf); - let mut ser = rmp_serde::Serializer::new(buf); + let mut ser = rmp_serde::Serializer::new(buf).with_struct_map(); core::hint::black_box(&s).serialize(&mut ser) }); } From de327a74b610ff2a0848306ae35257c2d36d5f13 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Sun, 7 Sep 2025 20:46:07 +0000 Subject: [PATCH 43/57] chore: default to lossless --- messagepack-serde/src/ser/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/messagepack-serde/src/ser/mod.rs b/messagepack-serde/src/ser/mod.rs index c7c7f05..bb959d7 100644 --- a/messagepack-serde/src/ser/mod.rs +++ b/messagepack-serde/src/ser/mod.rs @@ -48,7 +48,7 @@ pub fn to_slice(value: &T, buf: &mut [u8]) -> Result> where T: ser::Serialize + ?Sized, { - to_slice_with_config(value, buf, num::Exact) + to_slice_with_config(value, buf, num::LosslessMinimize) } /// Serialize value as messagepack with config. @@ -74,7 +74,7 @@ where T: ser::Serialize + ?Sized, W: std::io::Write, { - to_writer_with_config(value, writer, num::Exact) + to_writer_with_config(value, writer, num::LosslessMinimize) } #[cfg(feature = "std")] From cfd59b4007d863d81527bd933b7e3097b2eb9595 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Sun, 7 Sep 2025 20:52:05 +0000 Subject: [PATCH 44/57] test: fix to lossless --- messagepack-serde/src/ser/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/messagepack-serde/src/ser/mod.rs b/messagepack-serde/src/ser/mod.rs index bb959d7..4924a4e 100644 --- a/messagepack-serde/src/ser/mod.rs +++ b/messagepack-serde/src/ser/mod.rs @@ -534,9 +534,9 @@ mod tests { buf[..len], [ 0x93, // fixarr len = 3 - 0xd1, 0x00, 0x01, // int16 - 0xce, 0x00, 0x00, 0x00, 0x02, // uint32 - 0xd2, 0x00, 0x00, 0x00, 0x03, // uint32 + 0x01, // positive fixint 1 + 0x02, // positive fixint 2 + 0x03, // positive fixint 3 ] ); } From acd30f34d2d88f583d340379dc3f7fb607219ff7 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Sun, 7 Sep 2025 21:08:19 +0000 Subject: [PATCH 45/57] doc --- messagepack-serde/src/de/mod.rs | 2 ++ messagepack-serde/src/lib.rs | 2 -- messagepack-serde/src/ser/mod.rs | 40 +++++++++++++++++++++++++++++--- 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/messagepack-serde/src/de/mod.rs b/messagepack-serde/src/de/mod.rs index c66387d..c102644 100644 --- a/messagepack-serde/src/de/mod.rs +++ b/messagepack-serde/src/de/mod.rs @@ -1,3 +1,5 @@ +//! Deserialize support for messagepack + mod enum_; mod error; mod seq; diff --git a/messagepack-serde/src/lib.rs b/messagepack-serde/src/lib.rs index acc4d6c..d83f3ec 100644 --- a/messagepack-serde/src/lib.rs +++ b/messagepack-serde/src/lib.rs @@ -7,9 +7,7 @@ #[cfg(feature = "alloc")] extern crate alloc; -/// Deserialize support for messagepack pub mod de; -/// Serialize support for messagepack pub mod ser; mod value; diff --git a/messagepack-serde/src/ser/mod.rs b/messagepack-serde/src/ser/mod.rs index 4924a4e..27b9a5c 100644 --- a/messagepack-serde/src/ser/mod.rs +++ b/messagepack-serde/src/ser/mod.rs @@ -1,3 +1,37 @@ +//! Serialize support for messagepack +//! +//! ## Limitation +//! +//! MessagePack requires the length header of arrays and maps to be written +//! before any elements are encoded. Therefore this serializer needs serde +//! to provide the exact length up front. If serde calls +//! `serialize_seq(None)` or `serialize_map(None)`, this serializer returns +//! `Error::SeqLenNone`. +//! +//! Examples with `serde(flatten)`: +//! +//! ```rust +//! use serde::Serialize; +//! use std::collections::HashMap; +//! +//! // Fails +//! #[derive(Serialize)] +//! struct Inner { b: u8, c: u8 } +//! +//! #[derive(Serialize)] +//! struct Outer { +//! a: u8, +//! #[serde(flatten)] +//! extra: Inner, +//! } +//! +//! let mut buf = [0u8; 32]; +//! let v = Outer { a: 1, extra: Inner { b: 2, c: 3 } }; +//! let err = messagepack_serde::ser::to_slice(&v, &mut buf).unwrap_err(); +//! assert_eq!(err, messagepack_serde::ser::Error::SeqLenNone); +//! ``` +//! + mod error; mod map; mod num; @@ -569,9 +603,9 @@ mod tests { buf[..len], [ 0x93, // fixarr len = 3 - 0xd1, 0x00, 0x01, // int16 - 0xce, 0x00, 0x00, 0x00, 0x02, // uint32 - 0xd2, 0x00, 0x00, 0x00, 0x03, // uint32 + 0x01, // positive fixint 1 + 0x02, // positive fixint 2 + 0x03 // positive fixint 3 ] ); } From 7e92fe60e8839a8a46dec109a72460f6037190ff Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Sun, 7 Sep 2025 21:58:59 +0000 Subject: [PATCH 46/57] feat: recursion limit --- messagepack-serde/src/de/error.rs | 3 ++ messagepack-serde/src/de/mod.rs | 50 +++++++++++++++++++++++++++---- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/messagepack-serde/src/de/error.rs b/messagepack-serde/src/de/error.rs index e50efad..9d48972 100644 --- a/messagepack-serde/src/de/error.rs +++ b/messagepack-serde/src/de/error.rs @@ -7,6 +7,8 @@ pub(crate) type CoreError = messagepack_core::decode::Error; pub enum Error { /// Core error Decode(CoreError), + /// Recursion limit (nesting depth) exceeded + RecursionLimitExceeded, #[cfg(not(feature = "std"))] /// Parse error Custom, @@ -19,6 +21,7 @@ impl core::fmt::Display for Error { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { match self { Error::Decode(e) => e.fmt(f), + Error::RecursionLimitExceeded => write!(f, "recursion limit exceeded"), #[cfg(not(feature = "std"))] Error::Custom => write!(f, "Cannot deserialize format"), #[cfg(feature = "std")] diff --git a/messagepack-serde/src/de/mod.rs b/messagepack-serde/src/de/mod.rs index c102644..e387afa 100644 --- a/messagepack-serde/src/de/mod.rs +++ b/messagepack-serde/src/de/mod.rs @@ -34,14 +34,30 @@ where T::deserialize(&mut deserializer).map_err(std::io::Error::other) } +const MAX_RECURSION_DEPTH: usize = 256; + #[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)] struct Deserializer<'de> { input: &'de [u8], + depth: usize, } impl<'de> Deserializer<'de> { pub fn from_slice(input: &'de [u8]) -> Self { - Deserializer { input } + Deserializer { input, depth: 0 } + } + + fn recurse(&mut self, f: F) -> Result + where + F: FnOnce(&mut Self) -> V, + { + if self.depth == MAX_RECURSION_DEPTH { + return Err(Error::RecursionLimitExceeded); + } + self.depth += 1; + let result = f(self); + self.depth -= 1; + Ok(result) } fn decode>(&mut self) -> Result { @@ -74,7 +90,7 @@ impl<'de> Deserializer<'de> { } _ => return Err(CoreError::UnexpectedFormat.into()), }; - visitor.visit_seq(seq::FixLenAccess::new(self, n)) + self.recurse(move |des| visitor.visit_seq(seq::FixLenAccess::new(des, n)))? } fn decode_map_with_format(&mut self, format: Format, visitor: V) -> Result @@ -95,7 +111,7 @@ impl<'de> Deserializer<'de> { } _ => return Err(CoreError::UnexpectedFormat.into()), }; - visitor.visit_map(seq::FixLenAccess::new(self, n)) + self.recurse(move |des| visitor.visit_map(seq::FixLenAccess::new(des, n)))? } } @@ -224,18 +240,20 @@ impl<'de> de::Deserializer<'de> for &mut Deserializer<'de> { Ok(ident) => visitor.visit_enum(ident.into_deserializer()), _ => { let (format, rest) = Format::decode(self.input)?; - let mut des = Deserializer::from_slice(rest); + // inherit depth + des.depth = self.depth; let val = match format { Format::FixMap(_) | Format::Map16 | Format::Map32 | Format::FixArray(_) | Format::Array16 - | Format::Array32 => visitor.visit_enum(enum_::Enum::new(&mut des)), + | Format::Array32 => { + des.recurse(|d| visitor.visit_enum(enum_::Enum::new(d)))? + } _ => Err(CoreError::UnexpectedFormat.into()), }?; - self.input = des.input; Ok(val) @@ -259,6 +277,7 @@ mod tests { use rstest::rstest; use super::*; + use serde::de::IgnoredAny; #[rstest] #[case([0xc3],true)] @@ -359,4 +378,23 @@ mod tests { let decoded = from_slice::(buf.as_ref()).unwrap(); assert_eq!(decoded, expected); } + + #[test] + fn recursion_limit_ok_at_256() { + // [[[[...]]]] 256 nested array + let mut buf = vec![0x91u8; 256]; + buf.push(0xc0); + + let _ = from_slice::(&buf).unwrap(); + } + + #[test] + fn recursion_limit_err_over_256() { + // [[[[...]]]] 257 nested array + let mut buf = vec![0x91u8; 257]; + buf.push(0xc0); + + let err = from_slice::(&buf).unwrap_err(); + assert!(matches!(err, Error::RecursionLimitExceeded)); + } } From eac5d652b680195c32b42b72449ba31fe3f35939 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Mon, 8 Sep 2025 20:10:01 +0000 Subject: [PATCH 47/57] feat: impl usize, isize --- messagepack-core/src/decode/int.rs | 68 ++++++++++++++++++++++++++++++ messagepack-core/src/encode/int.rs | 37 +++++++++++++--- 2 files changed, 99 insertions(+), 6 deletions(-) diff --git a/messagepack-core/src/decode/int.rs b/messagepack-core/src/decode/int.rs index ded39d4..d3b1a30 100644 --- a/messagepack-core/src/decode/int.rs +++ b/messagepack-core/src/decode/int.rs @@ -74,6 +74,74 @@ impl_decode_int!(i16, Format::Int16); impl_decode_int!(i32, Format::Int32); impl_decode_int!(i64, Format::Int64); +impl<'a> Decode<'a> for u128 { + type Value = Self; + + fn decode(buf: &'a [u8]) -> Result<(Self::Value, &'a [u8])> { + let (val, buf) = u64::decode(buf)?; + Ok((Self::from(val), buf)) + } + + fn decode_with_format(format: Format, buf: &'a [u8]) -> Result<(Self::Value, &'a [u8])> { + let (val, buf) = u64::decode_with_format(format, buf)?; + Ok((Self::from(val), buf)) + } +} + +impl<'a> Decode<'a> for i128 { + type Value = Self; + + fn decode(buf: &'a [u8]) -> Result<(Self::Value, &'a [u8])> { + let (val, buf) = i64::decode(buf)?; + Ok((Self::from(val), buf)) + } + + fn decode_with_format(format: Format, buf: &'a [u8]) -> Result<(Self::Value, &'a [u8])> { + let (val, buf) = i64::decode_with_format(format, buf)?; + Ok((Self::from(val), buf)) + } +} + +impl<'a> Decode<'a> for usize { + type Value = Self; + + fn decode(buf: &'a [u8]) -> Result<(Self::Value, &'a [u8])> { + let (val, buf) = u64::decode(buf)?; + match usize::try_from(val) { + Ok(usize_val) => Ok((usize_val, buf)), + Err(_) => Err(Error::InvalidData), + } + } + + fn decode_with_format(format: Format, buf: &'a [u8]) -> Result<(Self::Value, &'a [u8])> { + let (val, buf) = u64::decode_with_format(format, buf)?; + match usize::try_from(val) { + Ok(usize_val) => Ok((usize_val, buf)), + Err(_) => Err(Error::InvalidData), + } + } +} + +impl<'a> Decode<'a> for isize { + type Value = Self; + + fn decode(buf: &'a [u8]) -> Result<(Self::Value, &'a [u8])> { + let (val, buf) = i64::decode(buf)?; + match isize::try_from(val) { + Ok(isize_val) => Ok((isize_val, buf)), + Err(_) => Err(Error::InvalidData), + } + } + + fn decode_with_format(format: Format, buf: &'a [u8]) -> Result<(Self::Value, &'a [u8])> { + let (val, buf) = i64::decode_with_format(format, buf)?; + match isize::try_from(val) { + Ok(isize_val) => Ok((isize_val, buf)), + Err(_) => Err(Error::InvalidData), + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/messagepack-core/src/encode/int.rs b/messagepack-core/src/encode/int.rs index 42a6032..c10f849 100644 --- a/messagepack-core/src/encode/int.rs +++ b/messagepack-core/src/encode/int.rs @@ -37,6 +37,18 @@ where } } +impl Encode for usize +where + W: IoWrite, +{ + fn encode(&self, writer: &mut W) -> Result::Error> { + match u64::try_from(*self) { + Ok(u64_uint) => u64_uint.encode(writer), + Err(_) => Err(Error::InvalidFormat), + } + } +} + impl Encode for i8 where W: IoWrite, @@ -71,12 +83,18 @@ macro_rules! impl_encode_int { } }; } -impl_encode_int!(u16, Format::Uint16, 3); -impl_encode_int!(u32, Format::Uint32, 5); -impl_encode_int!(u64, Format::Uint64, 9); -impl_encode_int!(i16, Format::Int16, 3); -impl_encode_int!(i32, Format::Int32, 5); -impl_encode_int!(i64, Format::Int64, 9); + +impl Encode for isize +where + W: IoWrite, +{ + fn encode(&self, writer: &mut W) -> Result { + match i64::try_from(*self) { + Ok(i64_int) => i64_int.encode(writer), + Err(_) => Err(Error::InvalidFormat), + } + } +} impl Encode for i128 where @@ -90,6 +108,13 @@ where } } +impl_encode_int!(u16, Format::Uint16, 3); +impl_encode_int!(u32, Format::Uint32, 5); +impl_encode_int!(u64, Format::Uint64, 9); +impl_encode_int!(i16, Format::Int16, 3); +impl_encode_int!(i32, Format::Int32, 5); +impl_encode_int!(i64, Format::Int64, 9); + /// encode minimum byte size #[derive(Debug, Copy, Clone, PartialOrd, Ord, PartialEq, Eq)] pub struct EncodeMinimizeInt(pub N); From 3678a104dded6b7797b4918ce82edb4b8c5826b9 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Mon, 8 Sep 2025 20:10:14 +0000 Subject: [PATCH 48/57] feat: impl tuple --- messagepack-core/src/decode/array.rs | 64 ++++++++++++++++++++++++++++ messagepack-core/src/encode/array.rs | 53 +++++++++++++++++++++++ 2 files changed, 117 insertions(+) diff --git a/messagepack-core/src/decode/array.rs b/messagepack-core/src/decode/array.rs index b3cf9f7..c5344c9 100644 --- a/messagepack-core/src/decode/array.rs +++ b/messagepack-core/src/decode/array.rs @@ -55,6 +55,70 @@ where } } +macro_rules! tuple_decode_impls { + ($($len:expr => ($($n:tt $name:ident)+))+ $(,)?) => { + $( + impl<'a, $($name),+> Decode<'a> for ($($name,)+) + where + $($name: Decode<'a>,)+ + { + type Value = ($(<$name as Decode<'a>>::Value,)+); + + fn decode(buf: &'a [u8]) -> Result<(Self::Value, &'a [u8])> { + let (format, buf) = Format::decode(buf)?; + match format { + Format::FixArray(_) | Format::Array16 | Format::Array32 => + Self::decode_with_format(format, buf), + _ => Err(Error::UnexpectedFormat), + } + } + + fn decode_with_format(format: Format, buf: &'a [u8]) + -> Result<(Self::Value, &'a [u8])> + { + let (len, mut p) = match format { + Format::FixArray(len) => (len.into(), buf), + Format::Array16 => NbyteReader::<2>::read(buf)?, + Format::Array32 => NbyteReader::<4>::read(buf)?, + _ => return Err(Error::UnexpectedFormat), + }; + if len != $len { + return Err(Error::InvalidData); + } + + let value = ( + $({ + let (v, next) = <$name as Decode<'a>>::decode(p)?; + p = next; + v + },)+ + ); + Ok((value, p)) + } + } + )+ + }; +} + +tuple_decode_impls! { + 1 => (0 V0) + 2 => (0 V0 1 V1) + 3 => (0 V0 1 V1 2 V2) + 4 => (0 V0 1 V1 2 V2 3 V3) + 5 => (0 V0 1 V1 2 V2 3 V3 4 V4) + 6 => (0 V0 1 V1 2 V2 3 V3 4 V4 5 V5) + 7 => (0 V0 1 V1 2 V2 3 V3 4 V4 5 V5 6 V6) + 8 => (0 V0 1 V1 2 V2 3 V3 4 V4 5 V5 6 V6 7 V7) + 9 => (0 V0 1 V1 2 V2 3 V3 4 V4 5 V5 6 V6 7 V7 8 V8) + 10 => (0 V0 1 V1 2 V2 3 V3 4 V4 5 V5 6 V6 7 V7 8 V8 9 V9) + 11 => (0 V0 1 V1 2 V2 3 V3 4 V4 5 V5 6 V6 7 V7 8 V8 9 V9 10 V10) + 12 => (0 V0 1 V1 2 V2 3 V3 4 V4 5 V5 6 V6 7 V7 8 V8 9 V9 10 V10 11 V11) + 13 => (0 V0 1 V1 2 V2 3 V3 4 V4 5 V5 6 V6 7 V7 8 V8 9 V9 10 V10 11 V11 12 V12) + 14 => (0 V0 1 V1 2 V2 3 V3 4 V4 5 V5 6 V6 7 V7 8 V8 9 V9 10 V10 11 V11 12 V12 13 V13) + 15 => (0 V0 1 V1 2 V2 3 V3 4 V4 5 V5 6 V6 7 V7 8 V8 9 V9 10 V10 11 V11 12 V12 13 V13 14 V14) + 16 => (0 V0 1 V1 2 V2 3 V3 4 V4 5 V5 6 V6 7 V7 8 V8 9 V9 10 V10 11 V11 12 V12 13 V13 14 V14 15 V15) +} + #[cfg(test)] mod tests { use super::*; diff --git a/messagepack-core/src/encode/array.rs b/messagepack-core/src/encode/array.rs index cc3cb64..62fc848 100644 --- a/messagepack-core/src/encode/array.rs +++ b/messagepack-core/src/encode/array.rs @@ -52,6 +52,59 @@ where } } +impl Encode for [V; N] +where + W: IoWrite, + V: Encode, +{ + fn encode(&self, writer: &mut W) -> Result::Error> { + self.as_slice().encode(writer) + } +} + +macro_rules! tuple_impls { + ($($len:expr => ($($n:tt $name:ident)+))+ $(,)?) => { + $( + tuple_impls!(@impl $len; $($n $name)+); + )+ + }; + (@impl $len:expr; $($n:tt $name:ident)+) => { + impl Encode for ($($name,)+) + where + W: IoWrite, + $($name: Encode,)+ + { + fn encode(&self, writer: &mut W) -> Result::Error> { + let format_len = ArrayFormatEncoder($len).encode(writer)?; + let mut array_len = 0; + $( + array_len += self.$n.encode(writer)?; + )+ + Ok(format_len + array_len) + } + } + }; +} + +tuple_impls! { + 1 => (0 V0) + 2 => (0 V0 1 V1) + 3 => (0 V0 1 V1 2 V2) + 4 => (0 V0 1 V1 2 V2 3 V3) + 5 => (0 V0 1 V1 2 V2 3 V3 4 V4) + 6 => (0 V0 1 V1 2 V2 3 V3 4 V4 5 V5) + 7 => (0 V0 1 V1 2 V2 3 V3 4 V4 5 V5 6 V6) + 8 => (0 V0 1 V1 2 V2 3 V3 4 V4 5 V5 6 V6 7 V7) + 9 => (0 V0 1 V1 2 V2 3 V3 4 V4 5 V5 6 V6 7 V7 8 V8) + 10 => (0 V0 1 V1 2 V2 3 V3 4 V4 5 V5 6 V6 7 V7 8 V8 9 V9) + 11 => (0 V0 1 V1 2 V2 3 V3 4 V4 5 V5 6 V6 7 V7 8 V8 9 V9 10 V10) + 12 => (0 V0 1 V1 2 V2 3 V3 4 V4 5 V5 6 V6 7 V7 8 V8 9 V9 10 V10 11 V11) + 13 => (0 V0 1 V1 2 V2 3 V3 4 V4 5 V5 6 V6 7 V7 8 V8 9 V9 10 V10 11 V11 12 V12) + 14 => (0 V0 1 V1 2 V2 3 V3 4 V4 5 V5 6 V6 7 V7 8 V8 9 V9 10 V10 11 V11 12 V12 13 V13) + 15 => (0 V0 1 V1 2 V2 3 V3 4 V4 5 V5 6 V6 7 V7 8 V8 9 V9 10 V10 11 V11 12 V12 13 V13 14 V14) + 16 => (0 V0 1 V1 2 V2 3 V3 4 V4 5 V5 6 V6 7 V7 8 V8 9 V9 10 V10 11 V11 12 V12 13 V13 14 V14 15 V15) +} + #[cfg(test)] mod tests { use super::*; From 4d5784415b6882bc48f86d21a29ce3f267f045c3 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Mon, 8 Sep 2025 20:11:04 +0000 Subject: [PATCH 49/57] refactor: use macro --- messagepack-core/src/encode/mod.rs | 41 ++++++++++++++++++------------ 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/messagepack-core/src/encode/mod.rs b/messagepack-core/src/encode/mod.rs index 8ba6497..8d6a651 100644 --- a/messagepack-core/src/encode/mod.rs +++ b/messagepack-core/src/encode/mod.rs @@ -59,24 +59,33 @@ where fn encode(&self, writer: &mut W) -> Result; } -impl Encode for &V -where - V: Encode, - W: IoWrite, -{ - fn encode(&self, writer: &mut W) -> Result::Error> { - Encode::encode(*self, writer) - } +macro_rules! deref_impl { + ( + $(#[$attr:meta])* + <$($desc:tt)+ + ) => { + $(#[$attr])* + impl<$($desc)+ + { + fn encode(&self, writer: &mut W) -> Result::Error> { + (**self).encode(writer) + } + } + }; } -impl Encode for &mut V -where - V: Encode, - W: IoWrite, -{ - fn encode(&self, writer: &mut W) -> Result::Error> { - Encode::encode(*self, writer) - } +deref_impl! { + Encode for &V + where + V: Encode, + W: IoWrite, +} + +deref_impl! { + Encode for &mut V + where + V: Encode, + W: IoWrite, } impl Encode for Format From 2953aafa9aeeffa597798abeb1cc52f9f31bdc5c Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Mon, 8 Sep 2025 21:24:16 +0000 Subject: [PATCH 50/57] feat: array decode --- messagepack-core/src/decode/array.rs | 57 ++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 15 deletions(-) diff --git a/messagepack-core/src/decode/array.rs b/messagepack-core/src/decode/array.rs index c5344c9..c5ffe86 100644 --- a/messagepack-core/src/decode/array.rs +++ b/messagepack-core/src/decode/array.rs @@ -33,25 +33,52 @@ where _ => return Err(Error::UnexpectedFormat), }; - let mut has_err = None; let mut buf_ptr = buf; - let collector = core::iter::repeat_n((), len).map_while(|_| match V::decode(buf_ptr) { - Ok((v, b)) => { - buf_ptr = b; - Some(v) - } - Err(e) => { - has_err = Some(e); - None + let out = (0..len) + .map(|_| { + let (v, next) = V::decode(buf_ptr)?; + buf_ptr = next; + Ok::<_, Error>(v) + }) + .collect::>()?; + Ok((out, buf_ptr)) + } +} + +impl<'a, const N: usize, V> Decode<'a> for [V; N] +where + V: Decode<'a>, +{ + type Value = [V::Value; N]; + fn decode(buf: &'a [u8]) -> Result<(Self::Value, &'a [u8])> { + let (format, buf) = Format::decode(buf)?; + match format { + Format::FixArray(_) | Format::Array16 | Format::Array32 => { + Self::decode_with_format(format, buf) } - }); - let res = Array::from_iter(collector); + _ => Err(Error::UnexpectedFormat), + } + } + fn decode_with_format(format: Format, buf: &'a [u8]) -> Result<(Self::Value, &'a [u8])> { + let (len, buf) = match format { + Format::FixArray(len) => (len.into(), buf), + Format::Array16 => NbyteReader::<2>::read(buf)?, + Format::Array32 => NbyteReader::<4>::read(buf)?, + _ => return Err(Error::UnexpectedFormat), + }; + if len != N { + return Err(Error::InvalidData); + }; - if let Some(e) = has_err { - Err(e) - } else { - Ok((res, buf_ptr)) + let mut tmp: [Option; N] = core::array::from_fn(|_| None); + let mut buf_ptr = buf; + for i in 0..N { + let (val, next) = V::decode(buf_ptr)?; + tmp[i] = Some(val); + buf_ptr = next } + let out = core::array::from_fn(|i| tmp[i].take().expect("initialized")); + Ok((out, buf_ptr)) } } From 5ae901f2ee75a9334dadfa0861f08e026f403abc Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Mon, 8 Sep 2025 21:24:23 +0000 Subject: [PATCH 51/57] test: add tests --- messagepack-core/src/decode/array.rs | 89 +++++++++++++++++++++++++--- 1 file changed, 82 insertions(+), 7 deletions(-) diff --git a/messagepack-core/src/decode/array.rs b/messagepack-core/src/decode/array.rs index c5ffe86..cb3acc5 100644 --- a/messagepack-core/src/decode/array.rs +++ b/messagepack-core/src/decode/array.rs @@ -149,14 +149,89 @@ tuple_decode_impls! { #[cfg(test)] mod tests { use super::*; - - #[test] - fn decode_int_array() { - let buf: &[u8] = &[0x95, 0x01, 0x02, 0x03, 0x04, 0x05]; + use rstest::rstest; + + #[rstest] + #[case(&[0x92, 0x01, 0x02, 0x01], vec![1u8, 2], &[0x01])] + #[case(&[0xdc, 0x00, 0x02, 0x2a, 0x2b], vec![42u8, 43], &[])] + fn array_decode_success( + #[case] buf: &[u8], + #[case] expect: Vec, + #[case] rest_expect: &[u8], + ) { let (decoded, rest) = ArrayDecoder::, u8>::decode(buf).unwrap(); - - let expect: &[u8] = &[1, 2, 3, 4, 5]; assert_eq!(decoded, expect); - assert_eq!(rest.len(), 0); + assert_eq!(rest, rest_expect); + } + + #[rstest] + fn array_decoder_unexpected_format() { + let buf = &[0x81, 0x01, 0x02]; // map(1) + let err = ArrayDecoder::, u8>::decode(buf).unwrap_err(); + assert!(matches!(err, Error::UnexpectedFormat)); + } + + #[rstest] + fn fixed_array_len0_success() { + let buf = &[0x90]; // array(0) + let (arr, rest) = <[u8; 0] as Decode>::decode(buf).unwrap(); + assert_eq!(arr, []); + assert!(rest.is_empty()); + } + + #[rstest] + fn fixed_array_len3_success() { + let buf = &[0x93, 0x0a, 0x0b, 0x0c]; + let (arr, rest) = <[u8; 3] as Decode>::decode(buf).unwrap(); + assert_eq!(arr, [10u8, 11, 12]); + assert!(rest.is_empty()); + } + + #[rstest] + #[case(&[0x92, 0x01, 0x02])] // len=2 + #[case(&[0x94, 0x01, 0x02, 0x03, 0x04])] // len=4 + fn fixed_array_len_mismatch(#[case] buf: &[u8]) { + let err = <[u8; 3] as Decode>::decode(buf).unwrap_err(); + assert!(matches!(err, Error::InvalidData)); + } + + #[rstest] + fn tuple1_success() { + let buf = &[0x91, 0x2a]; // [42] + let ((v0,), rest) = <(u8,) as Decode>::decode(buf).unwrap(); + assert_eq!(v0, 42); + assert!(rest.is_empty()); + } + + #[rstest] + #[case(&[0x92, 0x2a, 0x2b])] // fixarray + #[case(&[0xdc, 0x00, 0x02, 0x2a, 0x2b])] // array16(2) + fn tuple2_success(#[case] buf: &[u8]) { + let ((a, b), rest) = <(u8, u8) as Decode>::decode(buf).unwrap(); + assert_eq!((a, b), (42, 43)); + assert!(rest.is_empty()); + } + + #[rstest] + fn tuple3_success() { + let buf = &[0x93, 0x01, 0x02, 0x03]; + let ((a, b, c), rest) = <(u8, u8, u8) as Decode>::decode(buf).unwrap(); + assert_eq!((a, b, c), (1, 2, 3)); + assert!(rest.is_empty()); + } + + #[rstest] + #[case(&[0x92, 0x01, 0x02])] // len 2 + #[case(&[0xdc, 0x00, 0x04, 1, 2, 3, 4])] // len 4 + fn tuple_len_mismatch(#[case] buf: &[u8]) { + let err = <(u8, u8, u8) as Decode>::decode(buf).unwrap_err(); + assert!(matches!(err, Error::InvalidData)); + } + + #[rstest] + fn tuple_unexpected_format() { + let buf = &[0x81, 0x01, 0x02]; // map(1) + let err = <(u8,) as Decode>::decode(buf).unwrap_err(); + assert!(matches!(err, Error::UnexpectedFormat)); } } From 55d4ccb6e39cde483e5402e85124e41f1a982d48 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Mon, 8 Sep 2025 21:29:34 +0000 Subject: [PATCH 52/57] refactor: use iter_mut instead --- messagepack-core/src/decode/array.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/messagepack-core/src/decode/array.rs b/messagepack-core/src/decode/array.rs index cb3acc5..1785a7a 100644 --- a/messagepack-core/src/decode/array.rs +++ b/messagepack-core/src/decode/array.rs @@ -72,9 +72,9 @@ where let mut tmp: [Option; N] = core::array::from_fn(|_| None); let mut buf_ptr = buf; - for i in 0..N { + for item in tmp.iter_mut() { let (val, next) = V::decode(buf_ptr)?; - tmp[i] = Some(val); + *item = Some(val); buf_ptr = next } let out = core::array::from_fn(|i| tmp[i].take().expect("initialized")); From e89e17dddd917f77510938dbb2a80f92897c0efe Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Mon, 8 Sep 2025 21:42:07 +0000 Subject: [PATCH 53/57] test: encode tuple test --- messagepack-core/src/encode/array.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/messagepack-core/src/encode/array.rs b/messagepack-core/src/encode/array.rs index 62fc848..da54872 100644 --- a/messagepack-core/src/encode/array.rs +++ b/messagepack-core/src/encode/array.rs @@ -146,4 +146,14 @@ mod tests { assert_eq!(&buf, &expected); assert_eq!(n, expected.len()); } + + #[rstest] + #[case((1u8,), &[0x91,0x01])] + #[case((1u8,2u8), &[0x92,0x01,0x02])] + #[case((1u8,2u8,3u8), &[0x93,0x01,0x02,0x03])] + fn encode_tuple>>(#[case] v: V, #[case] expected: &[u8]) { + let mut buf = Vec::new(); + let _ = v.encode(&mut buf).unwrap(); + assert_eq!(buf, expected); + } } From 1a22e9d7a52e8018b7fc3e0a6db597b826ef7189 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Mon, 8 Sep 2025 21:57:43 +0000 Subject: [PATCH 54/57] chore: enable trusted publish --- .github/workflows/release.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 10b27c7..2287fbe 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -9,16 +9,20 @@ on: jobs: publish: runs-on: ubuntu-latest - + environment: release + permissions: + id-token: write steps: - name: Checkout branch uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Install Rust uses: moonrepo/setup-rust@ede6de059f8046a5e236c94046823e2af11ca670 # v1.2.2 + - uses: rust-lang/crates-io-auth-action@e919bc7605cde86df457cf5b93c5e103838bd879 # v1.0.1 + id: auth - name: Publish crates.io run: | cargo publish -p messagepack-core cargo publish -p messagepack-serde env: - CARGO_REGISTRY_TOKEN: ${{ secrets.CRATES_IO_API_TOKEN }} + CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }} \ No newline at end of file From 1f4cc099b6dfe8b154f78b98bf7d6dcfb9d856a8 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Mon, 8 Sep 2025 21:59:38 +0000 Subject: [PATCH 55/57] chore: bump version --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cd064b0..5d79557 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -299,7 +299,7 @@ dependencies = [ [[package]] name = "messagepack-core" -version = "0.1.2" +version = "0.1.3" dependencies = [ "num-traits", "rstest", @@ -307,7 +307,7 @@ dependencies = [ [[package]] name = "messagepack-serde" -version = "0.1.2" +version = "0.1.3" dependencies = [ "messagepack-core", "num-traits", diff --git a/Cargo.toml b/Cargo.toml index 5ace005..88ff3b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = ["messagepack-core", "messagepack-serde", "messagepack-bench"] resolver = "3" [workspace.package] -version = "0.1.2" +version = "0.1.3" edition = "2024" license = "Apache-2.0 OR MIT" repository = "https://github.com/tunamaguro/messagepack-rs" From 94692aa6a86efae700deb1cf19d3078a7933086c Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Mon, 8 Sep 2025 22:01:51 +0000 Subject: [PATCH 56/57] fix: add permission --- .github/workflows/codspeed.yaml | 3 +++ .github/workflows/coverage.yaml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/codspeed.yaml b/.github/workflows/codspeed.yaml index 13b6d1a..91e392f 100644 --- a/.github/workflows/codspeed.yaml +++ b/.github/workflows/codspeed.yaml @@ -1,5 +1,8 @@ name: Codspeed bench +permissions: + contents: read + on: pull_request: types: diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index 5ba96fd..37bf855 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -1,5 +1,8 @@ name: codecov +permissions: + contents: read + on: pull_request: types: From 8fe64552381c9ef8843710f6426d92d912e36054 Mon Sep 17 00:00:00 2001 From: tunamaguro Date: Mon, 8 Sep 2025 22:06:00 +0000 Subject: [PATCH 57/57] chore: add env --- .github/workflows/codspeed.yaml | 3 ++- .github/workflows/coverage.yaml | 2 ++ .github/workflows/pull_request.yaml | 6 ++++++ .github/workflows/release.yaml | 2 ++ 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codspeed.yaml b/.github/workflows/codspeed.yaml index 91e392f..4dc95e7 100644 --- a/.github/workflows/codspeed.yaml +++ b/.github/workflows/codspeed.yaml @@ -40,7 +40,8 @@ jobs: with: cache-target: release bins: cargo-codspeed - + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Build the benchmark target(s) run: cargo codspeed build diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index 37bf855..27742aa 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -41,6 +41,8 @@ jobs: cache-base: main components: llvm-tools-preview bins: cargo-llvm-cov + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Run test coverage run: | cargo llvm-cov --all-features --workspace --lcov --output-path lcov.info diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 8126cc4..a87b160 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -40,6 +40,8 @@ jobs: with: components: rustfmt cache-base: main + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Run format run: | cargo fmt --all --check @@ -54,6 +56,8 @@ jobs: with: components: clippy cache-base: main + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Run lint run: | RUSTFLAGS="--deny warnings" cargo clippy --workspace --all-targets --all-features @@ -70,6 +74,8 @@ jobs: uses: moonrepo/setup-rust@ede6de059f8046a5e236c94046823e2af11ca670 # v1.2.2 with: cache-base: main + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Run test run: | cargo test --all-features --workspace \ No newline at end of file diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2287fbe..e7abd5b 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -17,6 +17,8 @@ jobs: uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Install Rust uses: moonrepo/setup-rust@ede6de059f8046a5e236c94046823e2af11ca670 # v1.2.2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - uses: rust-lang/crates-io-auth-action@e919bc7605cde86df457cf5b93c5e103838bd879 # v1.0.1 id: auth - name: Publish crates.io