sys/dev/if_wg/module/crypto/zinc/chacha20/chacha20-arm-glue.c

   1 // SPDX-License-Identifier: GPL-2.0 OR MIT
   2 /*
   3  * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
   4  */
   5
   6 #include <asm/hwcap.h>
   7 #include <asm/neon.h>
   8 #if defined(CONFIG_ZINC_ARCH_ARM)
   9 #include <asm/system_info.h>
  10 #include <asm/cputype.h>
  11 #endif
  12
  13 asmlinkage void chacha20_arm(u8 *out, const u8 *in, const size_t len,
  14                              const u32 key[8], const u32 counter[4]);
  15 asmlinkage void hchacha20_arm(const u32 state[16], u32 out[8]);
  16 asmlinkage void chacha20_neon(u8 *out, const u8 *in, const size_t len,
  17                               const u32 key[8], const u32 counter[4]);
  18
  19 static bool chacha20_use_neon __ro_after_init;
  20 static bool *const chacha20_nobs[] __initconst = { &chacha20_use_neon };
  21 static void __init chacha20_fpu_init(void)
  22 {
  23 #if defined(CONFIG_ZINC_ARCH_ARM64)
  24         chacha20_use_neon = cpu_have_named_feature(ASIMD);
  25 #elif defined(CONFIG_ZINC_ARCH_ARM)
  26         switch (read_cpuid_part()) {
  27         case ARM_CPU_PART_CORTEX_A7:
  28         case ARM_CPU_PART_CORTEX_A5:
  29                 /* The Cortex-A7 and Cortex-A5 do not perform well with the NEON
  30                  * implementation but do incredibly with the scalar one and use
  31                  * less power.
  32                  */
  33                 break;
  34         default:
  35                 chacha20_use_neon = elf_hwcap & HWCAP_NEON;
  36         }
  37 #endif
  38 }
  39
  40 static inline bool chacha20_arch(struct chacha20_ctx *ctx, u8 *dst,
  41                                  const u8 *src, size_t len,
  42                                  simd_context_t *simd_context)
  43 {
  44         /* SIMD disables preemption, so relax after processing each page. */
  45         BUILD_BUG_ON(PAGE_SIZE < CHACHA20_BLOCK_SIZE ||
  46                      PAGE_SIZE % CHACHA20_BLOCK_SIZE);
  47
  48         for (;;) {
  49                 if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && chacha20_use_neon &&
  50                     len >= CHACHA20_BLOCK_SIZE * 3 && simd_use(simd_context)) {
  51                         const size_t bytes = min_t(size_t, len, PAGE_SIZE);
  52
  53                         chacha20_neon(dst, src, bytes, ctx->key, ctx->counter);
  54                         ctx->counter[0] += (bytes + 63) / 64;
  55                         len -= bytes;
  56                         if (!len)
  57                                 break;
  58                         dst += bytes;
  59                         src += bytes;
  60                         simd_relax(simd_context);
  61                 } else {
  62                         chacha20_arm(dst, src, len, ctx->key, ctx->counter);
  63                         ctx->counter[0] += (len + 63) / 64;
  64                         break;
  65                 }
  66         }
  67
  68         return true;
  69 }
  70
  71 static inline bool hchacha20_arch(u32 derived_key[CHACHA20_KEY_WORDS],
  72                                   const u8 nonce[HCHACHA20_NONCE_SIZE],
  73                                   const u8 key[HCHACHA20_KEY_SIZE],
  74                                   simd_context_t *simd_context)
  75 {
  76         if (IS_ENABLED(CONFIG_ZINC_ARCH_ARM)) {
  77                 u32 x[] = { CHACHA20_CONSTANT_EXPA,
  78                             CHACHA20_CONSTANT_ND_3,
  79                             CHACHA20_CONSTANT_2_BY,
  80                             CHACHA20_CONSTANT_TE_K,
  81                             get_unaligned_le32(key + 0),
  82                             get_unaligned_le32(key + 4),
  83                             get_unaligned_le32(key + 8),
  84                             get_unaligned_le32(key + 12),
  85                             get_unaligned_le32(key + 16),
  86                             get_unaligned_le32(key + 20),
  87                             get_unaligned_le32(key + 24),
  88                             get_unaligned_le32(key + 28),
  89                             get_unaligned_le32(nonce + 0),
  90                             get_unaligned_le32(nonce + 4),
  91                             get_unaligned_le32(nonce + 8),
  92                             get_unaligned_le32(nonce + 12)
  93                           };
  94                 hchacha20_arm(x, derived_key);
  95                 return true;
  96         }
  97         return false;
  98 }