Merge bitcoin-core/secp256k1#1492: tests: Add Wycheproof ECDH vectors

e266ba11ae tests: Add Wycheproof ECDH vectors (RandomLattice)

Pull request description:

ACKs for top commit:
  jonasnick:
    ACK e266ba11ae

Tree-SHA512: a5cc59886595b134dadcc50e6cd6f03ce036c2857cdd848f138f0c49d4bd742ae5eb5ebca7840ec8666b5d43fa9c4f67cde4d0fb2245b1cf56b079ca3f7c7f8e

Makefile.am

@@ -250,9 +250,17 @@ maintainer-clean-local: clean-precomp
 ### (see the comments in the previous section for detailed rationale)
 TESTVECTORS = src/wycheproof/ecdsa_secp256k1_sha256_bitcoin_test.h
 
+if ENABLE_MODULE_ECDH
+TESTVECTORS += src/wycheproof/ecdh_secp256k1_test.h
+endif
+
 src/wycheproof/ecdsa_secp256k1_sha256_bitcoin_test.h:
 	mkdir -p $(@D)
-	python3 $(top_srcdir)/tools/tests_wycheproof_generate.py $(top_srcdir)/src/wycheproof/ecdsa_secp256k1_sha256_bitcoin_test.json > $@
+	python3 $(top_srcdir)/tools/tests_wycheproof_generate_ecdsa.py $(top_srcdir)/src/wycheproof/ecdsa_secp256k1_sha256_bitcoin_test.json > $@
+
+src/wycheproof/ecdh_secp256k1_test.h:
+	mkdir -p $(@D)
+	python3 $(top_srcdir)/tools/tests_wycheproof_generate_ecdh.py $(top_srcdir)/src/wycheproof/ecdh_secp256k1_test.json > $@
 
 testvectors: $(TESTVECTORS)
 
@@ -276,7 +284,9 @@ EXTRA_DIST += sage/secp256k1_params.sage
 EXTRA_DIST += sage/weierstrass_prover.sage
 EXTRA_DIST += src/wycheproof/WYCHEPROOF_COPYING
 EXTRA_DIST += src/wycheproof/ecdsa_secp256k1_sha256_bitcoin_test.json
-EXTRA_DIST += tools/tests_wycheproof_generate.py
+EXTRA_DIST += src/wycheproof/ecdh_secp256k1_test.json
+EXTRA_DIST += tools/tests_wycheproof_generate_ecdsa.py
+EXTRA_DIST += tools/tests_wycheproof_generate_ecdh.py
 
 if ENABLE_MODULE_ECDH
 include src/modules/ecdh/Makefile.am.include

src/modules/ecdh/Makefile.am.include

@@ -2,3 +2,4 @@ include_HEADERS += include/secp256k1_ecdh.h
 noinst_HEADERS += src/modules/ecdh/main_impl.h
 noinst_HEADERS += src/modules/ecdh/tests_impl.h
 noinst_HEADERS += src/modules/ecdh/bench_impl.h
+noinst_HEADERS += src/wycheproof/ecdh_secp256k1_test.h

src/modules/ecdh/tests_impl.h

@@ -7,6 +7,13 @@
 #ifndef SECP256K1_MODULE_ECDH_TESTS_H
 #define SECP256K1_MODULE_ECDH_TESTS_H
 
+static int ecdh_hash_function_test_xpassthru(unsigned char *output, const unsigned char *x, const unsigned char *y, void *data) {
+    (void)y;
+    (void)data;
+    memcpy(output, x, 32);
+    return 1;
+}
+
 static int ecdh_hash_function_test_fail(unsigned char *output, const unsigned char *x, const unsigned char *y, void *data) {
     (void)output;
     (void)x;
@@ -142,11 +149,45 @@ static void test_result_basepoint(void) {
     }
 }
 
+static void test_ecdh_wycheproof(void) {
+#include "../../wycheproof/ecdh_secp256k1_test.h"
+    int t;
+    for (t = 0; t < SECP256K1_ECDH_WYCHEPROOF_NUMBER_TESTVECTORS; t++) {
+        int parsed_ok;
+        secp256k1_pubkey point;
+        const unsigned char *pk;
+        const unsigned char *sk;
+        const unsigned char *expected_shared_secret;
+        unsigned char output_ecdh[65] = { 0 };
+
+        int expected_result;
+
+        memset(&point, 0, sizeof(point));
+        pk = &wycheproof_ecdh_public_keys[testvectors[t].pk_offset];
+        parsed_ok = secp256k1_ec_pubkey_parse(CTX, &point, pk, testvectors[t].pk_len);
+
+        expected_result = testvectors[t].expected_result;
+        CHECK(parsed_ok == expected_result);
+        if (!parsed_ok) {
+            continue;
+        }
+
+        sk = &wycheproof_ecdh_private_keys[testvectors[t].sk_offset];
+        CHECK(testvectors[t].sk_len == 32);
+
+        CHECK(secp256k1_ecdh(CTX, output_ecdh, &point, sk, ecdh_hash_function_test_xpassthru, NULL) == 1);
+        expected_shared_secret = &wycheproof_ecdh_shared_secrets[testvectors[t].shared_offset];
+
+        CHECK(secp256k1_memcmp_var(output_ecdh, expected_shared_secret, testvectors[t].shared_len) == 0);
+    }
+}
+
 static void run_ecdh_tests(void) {
     test_ecdh_api();
     test_ecdh_generator_basepoint();
     test_bad_scalar();
     test_result_basepoint();
+    test_ecdh_wycheproof();
 }
 
 #endif /* SECP256K1_MODULE_ECDH_TESTS_H */

src/wycheproof/WYCHEPROOF_COPYING

@@ -3,9 +3,18 @@
   `b063b4aedae951c69df014cd25fa6d69ae9e8cb9`, see
   https://github.com/google/wycheproof/blob/b063b4aedae951c69df014cd25fa6d69ae9e8cb9/testvectors_v1/ecdsa_secp256k1_sha256_bitcoin_test.json
 
+* The file `ecdh_secp256k1_test.json` in this directory
+  comes from Google's project Wycheproof with git commit
+  `d9f6ec7d8bd8c96da05368999094e4a75ba5cb3d`, see
+  https://github.com/google/wycheproof/blob/d9f6ec7d8bd8c96da05368999094e4a75ba5cb3d/testvectors_v1/ecdh_secp256k1_test.json
+
 * The file `ecdsa_secp256k1_sha256_bitcoin_test.h` is generated from
   `ecdsa_secp256k1_sha256_bitcoin_test.json` using the script
-  `tests_wycheproof_generate.py`.
+  `tests_wycheproof_generate_ecdsa.py`.
+
+* The file `ecdh_secp256k1_test.h` is generated from
+  `ecdh_secp256k1_test.json` using the script
+  `tests_wycheproof_generate_ecdh.py`.
 
 -------------------------------------------------------------------------------
 

src/wycheproof/ecdsa_secp256k1_sha256_bitcoin_test.h

@@ -1,4 +1,4 @@
-/* Note: this file was autogenerated using tests_wycheproof_generate.py. Do not edit. */
+/* Note: this file was autogenerated using tests_wycheproof_generate_ecdsa.py. Do not edit. */
 #define SECP256K1_ECDSA_WYCHEPROOF_NUMBER_TESTVECTORS (463)
 
 typedef struct {

tools/tests_wycheproof_generate_ecdh.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+# Copyright (c) 2024 Random "Randy" Lattice and Sean Andersen
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or https://www.opensource.org/licenses/mit-license.php.
+'''
+Generate a C file with ECDH testvectors from the Wycheproof project.
+'''
+
+import json
+import sys
+
+from binascii import hexlify, unhexlify
+from wycheproof_utils import to_c_array
+
+def should_skip_flags(test_vector_flags):
+    # skip these vectors because they are for ASN.1 encoding issues and other curves.
+    # for more details, see https://github.com/bitcoin-core/secp256k1/pull/1492#discussion_r1572491546
+    flags_to_skip = {"InvalidAsn", "WrongCurve"}
+    return any(flag in test_vector_flags for flag in flags_to_skip)
+
+def should_skip_tcid(test_vector_tcid):
+    # We skip some test case IDs that have a public key whose custom ASN.1 representation explicitly
+    # encodes some curve parameters that are invalid. libsecp256k1 never parses this part so we do
+    # not care testing those. See https://github.com/bitcoin-core/secp256k1/pull/1492#discussion_r1572491546
+    tcids_to_skip = [496, 497, 502, 503, 504, 505, 507]
+    return test_vector_tcid in tcids_to_skip
+
+# Rudimentary ASN.1 DER public key parser.
+# This should not be used for anything other than parsing Wycheproof test vectors.
+def parse_der_pk(s):
+    tag = s[0]
+    L = int(s[1])
+    offset = 0
+    if L & 0x80:
+        if L == 0x81:
+            L = int(s[2])
+            offset = 1
+        elif L == 0x82:
+            L = 256 * int(s[2]) + int(s[3])
+            offset = 2
+        else:
+            raise ValueError("invalid L")
+    value = s[(offset + 2):(L + 2 + offset)]
+    rest = s[(L + 2 + offset):]
+
+    if len(rest) > 0 or tag == 0x06: # OBJECT IDENTIFIER
+        return parse_der_pk(rest)
+    if tag == 0x03: # BIT STRING
+        return value
+    if tag == 0x30: # SEQUENCE
+        return parse_der_pk(value)
+    raise ValueError("unknown tag")
+
+def parse_public_key(pk):
+    der_pub_key = parse_der_pk(unhexlify(pk))   # Convert back to str and strip off the `0x`
+    return hexlify(der_pub_key).decode()[2:]
+
+def normalize_private_key(sk):
+    # Ensure the private key is at most 64 characters long, retaining the last 64 if longer.
+    # In the wycheproof test vectors, some private keys have leading zeroes
+    normalized = sk[-64:].zfill(64)
+    if len(normalized) != 64:
+        raise ValueError("private key must be exactly 64 characters long.")
+    return normalized
+
+def normalize_expected_result(er):
+    result_mapping = {"invalid": 0, "valid": 1, "acceptable": 1}
+    return result_mapping[er]
+
+filename_input = sys.argv[1]
+
+with open(filename_input) as f:
+    doc = json.load(f)
+
+num_vectors = 0
+offset_sk_running, offset_pk_running, offset_shared = 0, 0, 0
+test_vectors_out = ""
+private_keys = ""
+shared_secrets = ""
+public_keys = ""
+cache_sks = {}
+cache_public_keys = {}
+
+for group in doc['testGroups']:
+    assert group["type"] == "EcdhTest"
+    assert group["curve"] == "secp256k1"
+    for test_vector in group['tests']:
+        if should_skip_flags(test_vector['flags']) or should_skip_tcid(test_vector['tcId']):
+            continue
+
+        public_key = parse_public_key(test_vector['public'])
+        private_key = normalize_private_key(test_vector['private'])
+        expected_result = normalize_expected_result(test_vector['result'])
+
+        # // 2 to convert hex to byte length
+        shared_size = len(test_vector['shared']) // 2
+        sk_size = len(private_key) // 2
+        pk_size = len(public_key) // 2
+
+        new_sk = False
+        sk = to_c_array(private_key)
+        sk_offset = offset_sk_running
+
+        # check for repeated sk
+        if sk not in cache_sks:
+            if num_vectors != 0 and sk_size != 0:
+                private_keys += ",\n  "
+            cache_sks[sk] = offset_sk_running
+            private_keys += sk
+            new_sk = True
+        else:
+            sk_offset = cache_sks[sk]
+
+        new_pk = False
+        pk = to_c_array(public_key) if public_key != '0x' else ''
+
+        pk_offset = offset_pk_running
+        # check for repeated pk
+        if pk not in cache_public_keys:
+            if num_vectors != 0 and len(pk) != 0:
+                public_keys += ",\n  "
+            cache_public_keys[pk] = offset_pk_running
+            public_keys += pk
+            new_pk = True
+        else:
+            pk_offset = cache_public_keys[pk]
+
+
+        shared_secrets += ",\n  " if num_vectors and shared_size else ""
+        shared_secrets += to_c_array(test_vector['shared'])
+        wycheproof_tcid = test_vector['tcId']
+
+        test_vectors_out += "  /" + "* tcId: " + str(test_vector['tcId']) + ". " + test_vector['comment'] + " *" + "/\n"
+        test_vectors_out += f"  {{{pk_offset}, {pk_size}, {sk_offset}, {sk_size}, {offset_shared}, {shared_size}, {expected_result}, {wycheproof_tcid} }},\n"
+        if new_sk:
+            offset_sk_running += sk_size
+        if new_pk:
+            offset_pk_running += pk_size
+        offset_shared += shared_size
+        num_vectors += 1
+
+struct_definition = """
+typedef struct {
+    size_t pk_offset;
+    size_t pk_len;
+    size_t sk_offset;
+    size_t sk_len;
+    size_t shared_offset;
+    size_t shared_len;
+    int expected_result;
+    int wycheproof_tcid;
+} wycheproof_ecdh_testvector;
+"""
+
+print("/* Note: this file was autogenerated using tests_wycheproof_ecdh.py. Do not edit. */")
+print(f"#define SECP256K1_ECDH_WYCHEPROOF_NUMBER_TESTVECTORS ({num_vectors})")
+
+print(struct_definition)
+
+print("static const unsigned char wycheproof_ecdh_private_keys[]    = { " + private_keys + "};\n")
+print("static const unsigned char wycheproof_ecdh_public_keys[] = { " + public_keys + "};\n")
+print("static const unsigned char wycheproof_ecdh_shared_secrets[]  = { " + shared_secrets + "};\n")
+
+print("static const wycheproof_ecdh_testvector testvectors[SECP256K1_ECDH_WYCHEPROOF_NUMBER_TESTVECTORS] = {")
+print(test_vectors_out)
+print("};")

tools/tests_wycheproof_generate_ecdsa.py

@@ -9,6 +9,8 @@ Generate a C file with ECDSA testvectors from the Wycheproof project.
 import json
 import sys
 
+from wycheproof_utils import to_c_array
+
 filename_input = sys.argv[1]
 
 with open(filename_input) as f:
@@ -16,12 +18,6 @@ with open(filename_input) as f:
 
 num_groups = len(doc['testGroups'])
 
-def to_c_array(x):
-    if x == "":
-        return ""
-    s = ',0x'.join(a+b for a,b in zip(x[::2], x[1::2]))
-    return "0x" + s
-
 
 num_vectors = 0
 offset_msg_running, offset_pk_running, offset_sig = 0, 0, 0
@@ -101,7 +97,7 @@ typedef struct {
 """
 
 
-print("/* Note: this file was autogenerated using tests_wycheproof_generate.py. Do not edit. */")
+print("/* Note: this file was autogenerated using tests_wycheproof_generate_ecdsa.py. Do not edit. */")
 print(f"#define SECP256K1_ECDSA_WYCHEPROOF_NUMBER_TESTVECTORS ({num_vectors})")
 
 print(struct_definition)

tools/wycheproof_utils.py

@@ -0,0 +1,12 @@
+# Copyright (c) 2024 Random "Randy" Lattice and Sean Andersen
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or https://www.opensource.org/licenses/mit-license.php.
+'''
+Utility functions for generating C files for testvectors from the Wycheproof project.
+'''
+
+def to_c_array(x):
+    if x == "":
+        return ""
+    s = ',0x'.join(a + b for a, b in zip(x[::2], x[1::2]))
+    return "0x" + s