apps: test pkeyutl -peerform option

The -peerform option of the pkeyutl app was not exercised by any test.
Extend the -derive peer key subtest to convert the peer public key to
DER and derive against it with -peerform DER, checking the shared secret
matches the one derived from the PEM peer key, and that reading the DER
peer key as PEM fails.

Assisted-by: Claude:claude-opus-4-8

Reviewed-by: Paul Dale <[email protected]>
Reviewed-by: Andrew Dinh <[email protected]>
Reviewed-by: Daniel Kubec <[email protected]>
MergeDate: Mon Jul 27 09:10:00 2026
(Merged from https://github.com/openssl/openssl/pull/31953)

1 files changed, 22 insertions, 1 deletions

diff --git a/test/recipes/20-test_pkeyutl.t b/test/recipes/20-test_pkeyutl.t
index dec9b4b1839..79e2629743b 100644
--- a/test/recipes/20-test_pkeyutl.t
+++ b/test/recipes/20-test_pkeyutl.t
@@ -451,7 +451,7 @@ SKIP: {
         my $ecpub = srctop_file("test", "testecpub-p256.pem");
         my $rsapub = srctop_file("test", "testrsapub.pem");
 
-        plan tests => 5;
+        plan tests => 8;
 
         # ECDH derive against a matching peer public key
         ok(run(app(['openssl', 'pkeyutl', '-derive',
@@ -459,6 +459,27 @@ SKIP: {
                     '-out', 'derive_secret.bin'])),
            "Derive shared secret with matching peer key");
 
+        # -peerform: load the peer public key from a DER file and check the
+        # derived secret matches the one derived from the PEM peer key.
+        my $ecpub_der = "peer-p256.der";
+        ok(run(app(['openssl', 'pkey', '-pubin', '-in', $ecpub,
+                    '-outform', 'DER', '-out', $ecpub_der])),
+           "Convert peer public key to DER");
+        ok(run(app(['openssl', 'pkeyutl', '-derive',
+                    '-inkey', $eckey, '-peerkey', $ecpub_der,
+                    '-peerform', 'DER', '-out', 'derive_secret_der.bin']))
+           && compare('derive_secret.bin', 'derive_secret_der.bin') == 0,
+           "Derive with DER peer key via -peerform matches the PEM result");
+
+        # -peerform mismatch: reading a DER peer key as PEM fails.
+        with({ exit_checker => sub { return shift == 1; } },
+            sub {
+                ok(run(app(['openssl', 'pkeyutl', '-derive',
+                            '-inkey', $eckey, '-peerkey', $ecpub_der,
+                            '-peerform', 'PEM'])),
+                   "Fail when -peerform does not match the peer key encoding");
+            });
+
         # setup_peer: peer key file cannot be loaded
         with({ exit_checker => sub { return shift == 1; } },
             sub {