security: per-packet handshake masks (eliminates correlation fingerprint)

Previously handshake obfuscation used a FIXED mask derived from
HMAC(obf_key, u64::MAX). This meant bytes [4..6] (noise_len XOR
fixed_mask) produced the SAME 2-byte value on every handshake from
the same access key — a correlation fingerprint for DPI.

Now BOTH data and handshake packets use the same payload-sampling
approach:
  mask = HMAC-SHA256(obf_key, payload_sample[0..32])

For data packets:   payload_sample = AEAD ciphertext (random per packet)
For handshake packets: payload_sample = Noise ephemeral key (random per connection)

Result: every single byte on the wire is cryptographically independent
across packets. No fixed patterns, no correlation between connections.

Wire analysis after this change:
- Packet sizes: random (84-182 for handshake, variable for data)
- All header bytes: unique per packet (XOR with unique HMAC mask)
- Payload bytes: AEAD ciphertext / Noise handshake (indistinguishable from random)
- No protocol signatures, no version fields, no magic bytes visible on wire

ostp-core/src/crypto/obfuscation.rs

@@ -23,48 +23,39 @@ pub fn derive_psk(access_key: &[u8]) -> [u8; 32] {
     psk
 }
 
-/// Derives a 6-byte handshake mask using HMAC-SHA256(key, nonce).
-/// Covers session_id (4 bytes) + noise_len (2 bytes) to prevent
-/// DPI from seeing a constant length field in the handshake header.
-fn derive_handshake_mask(key: &[u8; 8], nonce: u64) -> [u8; 6] {
-    let mut mac = HmacSha256::new_from_slice(key).expect("HMAC accepts any key length");
-    mac.update(&nonce.to_be_bytes());
-    let result = mac.finalize().into_bytes();
-    let mut mask = [0u8; 6];
-    mask.copy_from_slice(&result[..6]);
-    mask
-}
-
 /// Wire layout for DATA packets:
-///   [0..4]   = session_id XOR HMAC(obf_key, ciphertext_sample)[0..4]
+///   [0..4]   = session_id XOR mask[0..4]
-///   [4..12]  = nonce XOR HMAC(obf_key, ciphertext_sample)[4..12]
+///   [4..12]  = nonce XOR mask[4..12]
-///   [12..]   = AEAD ciphertext (at least 16 bytes tag)
+///   [12..]   = AEAD ciphertext
+///   mask = HMAC-SHA256(obf_key, ciphertext_sample[0..32])
 ///
-/// Because the ciphertext sample is different for every packet, the derived
+/// Wire layout for HANDSHAKE packets:
-/// mask is cryptographically random and independent for each packet.
+///   [0..6]   = (session_id || noise_len) XOR mask[0..6]
-/// Thus, both session_id and nonce are completely masked and indistinguishable
+///   [6..]    = noise_payload || random_padding
-/// from pure random noise on the wire.
+///   mask = HMAC-SHA256(obf_key, noise_payload_sample[0..32])
+///
+/// In both cases, the mask is derived from the payload that follows the header.
+/// Since the payload contains cryptographically random data (AEAD ciphertext
+/// or Noise ephemeral key), the mask is unique per packet, making the entire
+/// wire output indistinguishable from random noise.
 pub fn obfuscate_packet_inplace(raw: &mut [u8], key: &[u8; 8], is_handshake: bool) {
     if !is_handshake && raw.len() >= 12 {
         let header_len = 12;
         if raw.len() > header_len {
             let ciphertext = &raw[header_len..];
-            let mut sample = [0u8; 32];
+            let mask = derive_payload_mask(key, ciphertext);
-            let take_len = ciphertext.len().min(32);
-            sample[..take_len].copy_from_slice(&ciphertext[..take_len]);
 
-            let mut mac = HmacSha256::new_from_slice(key).expect("HMAC accepts any key length");
-            mac.update(&sample);
-            let mask_result = mac.finalize().into_bytes();
-
-            // Mask the entire 12-byte header (session_id + nonce)
             for i in 0..12 {
-                raw[i] ^= mask_result[i];
+                raw[i] ^= mask[i];
             }
         }
-    } else if raw.len() >= 6 {
+    } else if is_handshake && raw.len() > 6 {
-        // Handshake: mask session_id (4 bytes) + noise_len (2 bytes)
+        // Handshake: sample the Noise payload (starts at byte 6) to derive
-        let mask = derive_handshake_mask(key, u64::MAX);
+        // a per-packet mask. The Noise payload begins with a random ephemeral
+        // key, so the mask will be unique for every handshake.
+        let payload = &raw[6..];
+        let mask = derive_payload_mask(key, payload);
+
         for i in 0..6 {
             raw[i] ^= mask[i];
         }
@@ -78,24 +69,12 @@ pub fn deobfuscate_header_inplace(
     is_handshake: bool,
 ) {
     if !is_handshake {
-        let mut sample = [0u8; 32];
+        let mask = derive_payload_mask(key, ciphertext);
-        let take_len = ciphertext.len().min(32);
-        sample[..take_len].copy_from_slice(&ciphertext[..take_len]);
-
-        let mut mac = HmacSha256::new_from_slice(key).expect("HMAC accepts any key length");
-        mac.update(&sample);
-        let mask_result = mac.finalize().into_bytes();
-
-        // Unmask the entire 12-byte header
         for i in 0..12 {
-            header[i] ^= mask_result[i];
-        }
-    } else {
-        let mask = derive_handshake_mask(key, u64::MAX);
-        for i in 0..header.len().min(6) {
             header[i] ^= mask[i];
         }
     }
+    // Handshake deobfuscation is not done via this function — use deobfuscate_packet_inplace
 }
 
 pub fn deobfuscate_packet_inplace(raw: &mut [u8], key: &[u8; 8], is_handshake: bool) {
@@ -105,10 +84,30 @@ pub fn deobfuscate_packet_inplace(raw: &mut [u8], key: &[u8; 8], is_handshake: b
         header.copy_from_slice(header_slice);
         deobfuscate_header_inplace(&mut header, ciphertext, key, is_handshake);
         header_slice.copy_from_slice(&header);
-    } else if raw.len() >= 6 {
+    } else if is_handshake && raw.len() > 6 {
-        let mask = derive_handshake_mask(key, u64::MAX);
+        // Handshake: the payload (Noise data) starts at byte 6,
+        // and was NOT masked — only the header [0..6] was.
+        // Derive the same mask from the unmasked payload.
+        let payload = &raw[6..];
+        let mask = derive_payload_mask(key, payload);
+
         for i in 0..6 {
             raw[i] ^= mask[i];
         }
     }
 }
+
+/// Derives a 32-byte mask from a payload sample using HMAC-SHA256.
+/// Used by both data and handshake obfuscation to produce per-packet unique masks.
+fn derive_payload_mask(key: &[u8; 8], payload: &[u8]) -> [u8; 32] {
+    let mut sample = [0u8; 32];
+    let take_len = payload.len().min(32);
+    sample[..take_len].copy_from_slice(&payload[..take_len]);
+
+    let mut mac = HmacSha256::new_from_slice(key).expect("HMAC accepts any key length");
+    mac.update(&sample);
+    let result = mac.finalize().into_bytes();
+    let mut mask = [0u8; 32];
+    mask.copy_from_slice(&result);
+    mask
+}

ostp-server/src/dispatcher.rs

@@ -158,12 +158,12 @@ impl Dispatcher {
             let psk = ostp_core::crypto::derive_psk(candidate_key.as_bytes());
 
             // Decode the session_id using this key's obfuscation
-            // Handshake header is 6 bytes: [session_id:4][noise_len:2]
+            // The handshake mask is derived from the Noise payload at bytes [6..],
-            let mut header = [0u8; 6];
+            // so we must deobfuscate the full packet, not just the header.
-            if packet.len() < 6 { continue; }
+            if packet.len() < 7 { continue; }
-            header.copy_from_slice(&packet[0..6]);
+            let mut trial = packet.to_vec();
-            ostp_core::crypto::deobfuscate_packet_inplace(&mut header, &obf_key, true);
+            ostp_core::crypto::deobfuscate_packet_inplace(&mut trial, &obf_key, true);
-            let candidate_session_id = u32::from_be_bytes([header[0], header[1], header[2], header[3]]);
+            let candidate_session_id = u32::from_be_bytes([trial[0], trial[1], trial[2], trial[3]]);
 
             let mut cfg = self.machine_config.clone();
             cfg.session_id = candidate_session_id;