Add ZWUS-7 support and default encoding

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "zwus"
-version = "0.2.0"
+version = "0.3.0"
 edition = "2021"
 rust-version = "1.70"
 description = "Zero Width Unicode Steganography — hide text in invisible characters."

README.md

@@ -4,7 +4,7 @@ Zero Width Unicode Steganography — hide text inside invisible characters.
 
 ```toml
 [dependencies]
-zwus = "0.2"
+zwus = "0.3"
 ```
 
 ## Usage
@@ -25,14 +25,14 @@ assert_eq!(decoded, vec![72, 101, 108]);
 
 ### Base
 
-Higher base = shorter output, but more likely visible in some renderers.
+Base 7 ranks printable ASCII so common English characters use one or two digits. Base 6 keeps Unicode code points as numbers. Some platforms strip particular zero-width characters.
 
 ```rust
 use zwus::Zwus;
 
 Zwus::encode_string_with_base("hi", 3); // safest
-Zwus::encode_string_with_base("hi", 6); // default, compact
+Zwus::encode_string_with_base("hi", 6); // code point encoding
-Zwus::encode_string_with_base("hi", 8); // most compact
+Zwus::encode_string_with_base("hi", 7); // default, compact for ordinary English text
 ```
 
 Decode must match the encode base:
@@ -44,6 +44,8 @@ let encoded = Zwus::encode_string_with_base("hi", 6);
 Zwus::decode_to_string_with_base(&encoded, 6);
 ```
 
+Number arrays use ordinary base digits in every standard. Base 7's frequency ranking applies only to strings. Decoding ignores visible text mixed into a payload.
+
 ### Embedded in visible text
 
 Non-ZWUS characters are automatically ignored during decoding, so hidden payloads survive being mixed into normal text.

src/lib.rs

@@ -1,7 +1,7 @@
 #![doc = include_str!("../README.md")]
 
-pub const DEFAULT_BASE: u8 = 6;
+pub const DEFAULT_BASE: u8 = 7;
-pub const SUPPORTED_BASES: [u8; 3] = [3, 6, 8];
+pub const SUPPORTED_BASES: [u8; 3] = [3, 6, 7];
 
 #[derive(Clone, Copy)]
 struct Alphabet {
@@ -21,11 +21,10 @@ const BASE_6: Alphabet = Alphabet {
     ],
 };
 
-const BASE_8: Alphabet = Alphabet {
+const BASE_7: Alphabet = Alphabet {
     unifier: '\u{200C}',
     digits: &[
-        '\u{200D}', '\u{200F}', '\u{00AD}', '\u{2060}', '\u{200B}', '\u{200E}', '\u{180E}',
+        '\u{200D}', '\u{200F}', '\u{00AD}', '\u{2060}', '\u{200B}', '\u{200E}', '\u{FEFF}',
-        '\u{FEFF}',
     ],
 };
 
@@ -34,8 +33,8 @@ fn alphabet(base: u8) -> Alphabet {
     match base {
         3 => BASE_3,
         6 => BASE_6,
-        8 => BASE_8,
+        7 => BASE_7,
-        _ => panic!("Unsupported base {base}. Use 3, 6, or 8."),
+        _ => panic!("Unsupported base {base}. Use 3, 6, or 7."),
     }
 }
 
@@ -98,52 +97,99 @@ fn decode_numbers(text: &str, base: u8) -> Vec<u32> {
         .collect()
 }
 
+const PRIORITY: &str = "te aoinshrdlucmfwypvbgkjqxz.,!?'-:;()0123456789ETAOINSHRDLUCMFWYPVBGKJQXZ";
+
+fn ascii_order() -> Vec<char> {
+    let mut order = Vec::new();
+    for c in PRIORITY
+        .chars()
+        .chain((32..=126).filter_map(char::from_u32))
+    {
+        if !order.contains(&c) {
+            order.push(c);
+        }
+    }
+    order
+}
+
+fn rank_of(c: char, order: &[char]) -> u32 {
+    order
+        .iter()
+        .position(|&x| x == c)
+        .map(|i| i as u32)
+        .unwrap_or_else(|| {
+            if (c as u32) < 32 {
+                c as u32 + 95
+            } else {
+                c as u32
+            }
+        })
+}
+
+fn point_of(rank: u32, order: &[char]) -> Option<char> {
+    if rank < 95 {
+        order.get(rank as usize).copied()
+    } else {
+        char::from_u32(if rank < 127 { rank - 95 } else { rank })
+    }
+}
+
 /// Zero Width Unicode Standard (ZWUS)
 pub struct Zwus;
 
 impl Zwus {
-    /// Encode a string using base 6 (default/compact).
+    /// Encode a string using base 7 (default/compact).
     pub fn encode_string(text: &str) -> String {
         Self::encode_string_with_base(text, DEFAULT_BASE)
     }
 
-    /// Encode a string using base 3, 6, or 8.
+    /// Encode a string using base 3, 6, or 7.
     pub fn encode_string_with_base(text: &str, base: u8) -> String {
-        encode_numbers(text.chars().map(|c| c as u32), base)
+        if base == 7 {
+            let order = ascii_order();
+            encode_numbers(text.chars().map(|c| rank_of(c, &order)), base)
+        } else {
+            encode_numbers(text.chars().map(|c| c as u32), base)
+        }
     }
 
-    /// Encode numbers using base 6 (default/compact).
+    /// Encode numbers using base 7 (default/compact).
     pub fn encode_number_array(numbers: &[u32]) -> String {
         Self::encode_number_array_with_base(numbers, DEFAULT_BASE)
     }
 
-    /// Encode numbers using base 3, 6, or 8.
+    /// Encode numbers using base 3, 6, or 7.
     pub fn encode_number_array_with_base(numbers: &[u32], base: u8) -> String {
         encode_numbers(numbers.iter().copied(), base)
     }
 
-    /// Decode to string using base 6 (default/compact).
+    /// Decode to string using base 7 (default/compact).
     /// Non-ZWUS chars are ignored automatically.
     pub fn decode_to_string(text: &str) -> String {
         Self::decode_to_string_with_base(text, DEFAULT_BASE)
     }
 
-    /// Decode to string using base 3, 6, or 8.
+    /// Decode to string using base 3, 6, or 7.
     /// Non-ZWUS chars are ignored automatically.
     pub fn decode_to_string_with_base(text: &str, base: u8) -> String {
+        let order = (base == 7).then(ascii_order);
         decode_numbers(text, base)
             .into_iter()
-            .filter_map(char::from_u32)
+            .filter_map(|n| {
+                order
+                    .as_ref()
+                    .map_or_else(|| char::from_u32(n), |o| point_of(n, o))
+            })
             .collect()
     }
 
-    /// Decode to numbers using base 6 (default/compact).
+    /// Decode to numbers using base 7 (default/compact).
     /// Non-ZWUS chars are ignored automatically.
     pub fn decode_to_number_array(text: &str) -> Vec<u32> {
         Self::decode_to_number_array_with_base(text, DEFAULT_BASE)
     }
 
-    /// Decode to numbers using base 3, 6, or 8.
+    /// Decode to numbers using base 3, 6, or 7.
     /// Non-ZWUS chars are ignored automatically.
     pub fn decode_to_number_array_with_base(text: &str, base: u8) -> Vec<u32> {
         decode_numbers(text, base)

tests/zwus.rs

@@ -1,11 +1,11 @@
 use zwus::Zwus;
 
 #[test]
-fn default_base_is_6() {
+fn default_base_is_7() {
     let text = "secret 🦀";
     let numbers = [0, 72, 101];
-    let encoded_text = Zwus::encode_string_with_base(text, 6);
+    let encoded_text = Zwus::encode_string_with_base(text, 7);
-    let encoded_numbers = Zwus::encode_number_array_with_base(&numbers, 6);
+    let encoded_numbers = Zwus::encode_number_array_with_base(&numbers, 7);
 
     assert_eq!(Zwus::encode_string(text), encoded_text);
     assert_eq!(Zwus::decode_to_string(&encoded_text), text);
@@ -14,9 +14,9 @@ fn default_base_is_6() {
 }
 
 #[test]
-fn roundtrip_string_base_3_6_8() {
+fn roundtrip_string_base_3_6_7() {
     let text = "secret 🦀 unicode";
-    for base in [3u8, 6, 8] {
+    for base in [3u8, 6, 7] {
         let encoded = Zwus::encode_string_with_base(text, base);
         let decoded = Zwus::decode_to_string_with_base(&encoded, base);
         assert_eq!(decoded, text);
@@ -24,9 +24,9 @@ fn roundtrip_string_base_3_6_8() {
 }
 
 #[test]
-fn roundtrip_number_array_base_3_6_8() {
+fn roundtrip_number_array_base_3_6_7() {
     let nums = [0u32, 72, 101, 108, 108, 111, 0x10FFFF];
-    for base in [3u8, 6, 8] {
+    for base in [3u8, 6, 7] {
         let encoded = Zwus::encode_number_array_with_base(&nums, base);
         let decoded = Zwus::decode_to_number_array_with_base(&encoded, base);
         assert_eq!(decoded, nums);
@@ -38,6 +38,11 @@ fn decode_ignores_non_zwus_characters() {
     let hidden = Zwus::encode_string_with_base("secret", 6);
     let mixed = format!("te{hidden}xt");
     assert_eq!(Zwus::decode_to_string_with_base(&mixed, 6), "secret");
+    let hidden = Zwus::encode_string_with_base("secret", 7);
+    assert_eq!(
+        Zwus::decode_to_string_with_base(&format!("te{hidden}xt"), 7),
+        "secret"
+    );
 }
 
 #[test]
@@ -45,12 +50,12 @@ fn npm_vectors_decode_to_test123_checkmark() {
     // Paste encoded outputs from your npm package here:
     //   zwus.encodeString("Test123!✅", 3)
     //   zwus.encodeString("Test123!✅", 6)
-    //   zwus.encodeString("Test123!✅", 8)
+    //   zwus.encodeString("Test123!✅", 7)
     //
     // Keep these as raw strings to preserve invisible chars exactly.
     let npm_base_3 = r#"​᠎᠎​᠎­​᠎‍᠎‍­​​᠎‍​­​​᠎‍‍­​‍​​­​‍​‍­​‍‍᠎­​᠎‍᠎­​​​‍᠎᠎‍‍‍"#;
     let npm_base_6 = r#"­­‍‌­​‎‌⁠‏‏‌⁠‏­‌‏­‏‌‏­­‌‏­⁠‌‎⁠‌‏‏​‏­‎"#;
-    let npm_base_8 = r#"‏­​‌‏​‎‌‏᠎⁠‌‏᠎​‌᠎‏‌᠎­‌᠎⁠‌​‏‌­⁠​‍‎"#;
+    let npm_base_7 = "\u{feff}\u{feff}\u{200c}\u{200f}\u{200c}\u{200f}\u{200d}\u{200c}\u{200d}\u{200c}\u{200e}\u{2060}\u{200c}\u{200e}\u{200b}\u{200c}\u{200e}\u{200e}\u{200c}\u{200b}\u{200f}\u{200c}\u{200b}\u{200f}\u{200d}\u{feff}\u{200d}";
 
     let expected = "Test123!✅";
 
@@ -63,11 +68,12 @@ fn npm_vectors_decode_to_test123_checkmark() {
         "Paste base-6 encoded payload into npm_base_6"
     );
     assert!(
-        !npm_base_8.is_empty(),
+        !npm_base_7.is_empty(),
-        "Paste base-8 encoded payload into npm_base_8"
+        "Paste base-7 encoded payload into npm_base_7"
     );
 
     assert_eq!(Zwus::decode_to_string_with_base(npm_base_3, 3), expected);
     assert_eq!(Zwus::decode_to_string_with_base(npm_base_6, 6), expected);
-    assert_eq!(Zwus::decode_to_string_with_base(npm_base_8, 8), expected);
+    assert_eq!(Zwus::decode_to_string_with_base(npm_base_7, 7), expected);
+    assert_eq!(Zwus::encode_string_with_base(expected, 7), npm_base_7);
 }