sha256.c (13195B)
1 #include "core/sha256.h" 2 3 #include <string.h> 4 5 /* Use the AArch64 SHA-256 crypto extension when the compiler knows the host 6 * has it (always true on Apple Silicon). SHA256_K is shared: the scalar core 7 * indexes it and the hardware core loads round-key vectors from it. */ 8 #if defined(__aarch64__) && defined(__ARM_FEATURE_SHA2) 9 #define SHA256_HW 1 10 #endif 11 12 static const u32 SHA256_K[64] = { 13 0x428a2f98u, 0x71374491u, 0xb5c0fbcfu, 0xe9b5dba5u, 0x3956c25bu, 14 0x59f111f1u, 0x923f82a4u, 0xab1c5ed5u, 0xd807aa98u, 0x12835b01u, 15 0x243185beu, 0x550c7dc3u, 0x72be5d74u, 0x80deb1feu, 0x9bdc06a7u, 16 0xc19bf174u, 0xe49b69c1u, 0xefbe4786u, 0x0fc19dc6u, 0x240ca1ccu, 17 0x2de92c6fu, 0x4a7484aau, 0x5cb0a9dcu, 0x76f988dau, 0x983e5152u, 18 0xa831c66du, 0xb00327c8u, 0xbf597fc7u, 0xc6e00bf3u, 0xd5a79147u, 19 0x06ca6351u, 0x14292967u, 0x27b70a85u, 0x2e1b2138u, 0x4d2c6dfcu, 20 0x53380d13u, 0x650a7354u, 0x766a0abbu, 0x81c2c92eu, 0x92722c85u, 21 0xa2bfe8a1u, 0xa81a664bu, 0xc24b8b70u, 0xc76c51a3u, 0xd192e819u, 22 0xd6990624u, 0xf40e3585u, 0x106aa070u, 0x19a4c116u, 0x1e376c08u, 23 0x2748774cu, 0x34b0bcb5u, 0x391c0cb3u, 0x4ed8aa4au, 0x5b9cca4fu, 24 0x682e6ff3u, 0x748f82eeu, 0x78a5636fu, 0x84c87814u, 0x8cc70208u, 25 0x90befffau, 0xa4506cebu, 0xbef9a3f7u, 0xc67178f2u}; 26 27 #ifndef SHA256_HW 28 static u32 sha256_rotr32(u32 v, u32 n) { return (v >> n) | (v << (32 - n)); } 29 #endif 30 31 void sha256_init(Sha256* s) { 32 s->h[0] = 0x6a09e667u; 33 s->h[1] = 0xbb67ae85u; 34 s->h[2] = 0x3c6ef372u; 35 s->h[3] = 0xa54ff53au; 36 s->h[4] = 0x510e527fu; 37 s->h[5] = 0x9b05688cu; 38 s->h[6] = 0x1f83d9abu; 39 s->h[7] = 0x5be0cd19u; 40 s->buflen = 0; 41 s->total = 0; 42 } 43 44 #ifndef SHA256_HW 45 static void sha256_block(Sha256* s, const u8* p) { 46 u32 w[64]; 47 for (u32 i = 0; i < 16; ++i) 48 w[i] = ((u32)p[i * 4] << 24) | ((u32)p[i * 4 + 1] << 16) | 49 ((u32)p[i * 4 + 2] << 8) | (u32)p[i * 4 + 3]; 50 for (u32 i = 16; i < 64; ++i) { 51 u32 s0 = sha256_rotr32(w[i - 15], 7) ^ sha256_rotr32(w[i - 15], 18) ^ 52 (w[i - 15] >> 3); 53 u32 s1 = sha256_rotr32(w[i - 2], 17) ^ sha256_rotr32(w[i - 2], 19) ^ 54 (w[i - 2] >> 10); 55 w[i] = w[i - 16] + s0 + w[i - 7] + s1; 56 } 57 u32 a = s->h[0], b = s->h[1], cc = s->h[2], d = s->h[3]; 58 u32 e = s->h[4], f = s->h[5], g = s->h[6], hh = s->h[7]; 59 for (u32 i = 0; i < 64; ++i) { 60 u32 S1 = sha256_rotr32(e, 6) ^ sha256_rotr32(e, 11) ^ sha256_rotr32(e, 25); 61 u32 ch = (e & f) ^ ((~e) & g); 62 u32 t1 = hh + S1 + ch + SHA256_K[i] + w[i]; 63 u32 S0 = sha256_rotr32(a, 2) ^ sha256_rotr32(a, 13) ^ sha256_rotr32(a, 22); 64 u32 mj = (a & b) ^ (a & cc) ^ (b & cc); 65 u32 t2 = S0 + mj; 66 hh = g; 67 g = f; 68 f = e; 69 e = d + t1; 70 d = cc; 71 cc = b; 72 b = a; 73 a = t1 + t2; 74 } 75 s->h[0] += a; 76 s->h[1] += b; 77 s->h[2] += cc; 78 s->h[3] += d; 79 s->h[4] += e; 80 s->h[5] += f; 81 s->h[6] += g; 82 s->h[7] += hh; 83 } 84 #endif /* !SHA256_HW */ 85 86 /* Absorb `nblocks` complete 64-byte blocks starting at `p`. 87 * 88 * On AArch64 hosts that the compiler knows have the SHA-256 crypto extension 89 * (Apple Silicon always does), this dispatches to a vectorized inline-asm core 90 * that is ~5-8x the scalar throughput -- the same hardware path ld64 uses, and 91 * the reason kit's Mach-O code-signature hash (a SHA-256 over every output 92 * page) stops dominating large links. Output is identical to the scalar core; 93 * only the instruction selection differs. Other hosts (and AArch64 builds 94 * without the feature compiled in) use the portable block below. Linux/FreeBSD 95 * AArch64 could gain the same speedup behind a runtime HWCAP probe -- deferred 96 * to keep core free of platform feature-detection. */ 97 #ifdef SHA256_HW 98 /* The fully unrolled 64-round asm template is one string literal past C99's 99 * 4095-char minimum; clang/gcc both accept it, so silence -Wpedantic here. */ 100 #pragma GCC diagnostic push 101 #pragma GCC diagnostic ignored "-Woverlength-strings" 102 static void sha256_blocks(Sha256* s, const u8* p, size_t nblocks) { 103 if (!nblocks) return; 104 __asm__ volatile( 105 "ld1 {v0.4s, v1.4s}, [%[st]] \n" /* ABCD / EFGH */ 106 "1: \n" 107 "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%[d]], #64\n" /* MSG0..MSG3 */ 108 "mov v2.16b, v0.16b \n" /* save ABCD */ 109 "mov v3.16b, v1.16b \n" /* save EFGH */ 110 "rev32 v4.16b, v4.16b \n" 111 "rev32 v5.16b, v5.16b \n" 112 "rev32 v6.16b, v6.16b \n" 113 "rev32 v7.16b, v7.16b \n" 114 "mov x8, %[k] \n" 115 "ld1 {v19.4s}, [x8], #16 \n" 116 "add v16.4s, v4.4s, v19.4s \n" /* TMP0 = M0+K0 */ 117 /* rounds 0-3 */ 118 "sha256su0 v4.4s, v5.4s \n" 119 "mov v18.16b, v0.16b \n" 120 "ld1 {v19.4s}, [x8], #16 \n" 121 "add v17.4s, v5.4s, v19.4s \n" 122 "sha256h q0, q1, v16.4s \n" 123 "sha256h2 q1, q18, v16.4s \n" 124 "sha256su1 v4.4s, v6.4s, v7.4s \n" 125 /* rounds 4-7 */ 126 "sha256su0 v5.4s, v6.4s \n" 127 "mov v18.16b, v0.16b \n" 128 "ld1 {v19.4s}, [x8], #16 \n" 129 "add v16.4s, v6.4s, v19.4s \n" 130 "sha256h q0, q1, v17.4s \n" 131 "sha256h2 q1, q18, v17.4s \n" 132 "sha256su1 v5.4s, v7.4s, v4.4s \n" 133 /* rounds 8-11 */ 134 "sha256su0 v6.4s, v7.4s \n" 135 "mov v18.16b, v0.16b \n" 136 "ld1 {v19.4s}, [x8], #16 \n" 137 "add v17.4s, v7.4s, v19.4s \n" 138 "sha256h q0, q1, v16.4s \n" 139 "sha256h2 q1, q18, v16.4s \n" 140 "sha256su1 v6.4s, v4.4s, v5.4s \n" 141 /* rounds 12-15 */ 142 "sha256su0 v7.4s, v4.4s \n" 143 "mov v18.16b, v0.16b \n" 144 "ld1 {v19.4s}, [x8], #16 \n" 145 "add v16.4s, v4.4s, v19.4s \n" 146 "sha256h q0, q1, v17.4s \n" 147 "sha256h2 q1, q18, v17.4s \n" 148 "sha256su1 v7.4s, v5.4s, v6.4s \n" 149 /* rounds 16-19 */ 150 "sha256su0 v4.4s, v5.4s \n" 151 "mov v18.16b, v0.16b \n" 152 "ld1 {v19.4s}, [x8], #16 \n" 153 "add v17.4s, v5.4s, v19.4s \n" 154 "sha256h q0, q1, v16.4s \n" 155 "sha256h2 q1, q18, v16.4s \n" 156 "sha256su1 v4.4s, v6.4s, v7.4s \n" 157 /* rounds 20-23 */ 158 "sha256su0 v5.4s, v6.4s \n" 159 "mov v18.16b, v0.16b \n" 160 "ld1 {v19.4s}, [x8], #16 \n" 161 "add v16.4s, v6.4s, v19.4s \n" 162 "sha256h q0, q1, v17.4s \n" 163 "sha256h2 q1, q18, v17.4s \n" 164 "sha256su1 v5.4s, v7.4s, v4.4s \n" 165 /* rounds 24-27 */ 166 "sha256su0 v6.4s, v7.4s \n" 167 "mov v18.16b, v0.16b \n" 168 "ld1 {v19.4s}, [x8], #16 \n" 169 "add v17.4s, v7.4s, v19.4s \n" 170 "sha256h q0, q1, v16.4s \n" 171 "sha256h2 q1, q18, v16.4s \n" 172 "sha256su1 v6.4s, v4.4s, v5.4s \n" 173 /* rounds 28-31 */ 174 "sha256su0 v7.4s, v4.4s \n" 175 "mov v18.16b, v0.16b \n" 176 "ld1 {v19.4s}, [x8], #16 \n" 177 "add v16.4s, v4.4s, v19.4s \n" 178 "sha256h q0, q1, v17.4s \n" 179 "sha256h2 q1, q18, v17.4s \n" 180 "sha256su1 v7.4s, v5.4s, v6.4s \n" 181 /* rounds 32-35 */ 182 "sha256su0 v4.4s, v5.4s \n" 183 "mov v18.16b, v0.16b \n" 184 "ld1 {v19.4s}, [x8], #16 \n" 185 "add v17.4s, v5.4s, v19.4s \n" 186 "sha256h q0, q1, v16.4s \n" 187 "sha256h2 q1, q18, v16.4s \n" 188 "sha256su1 v4.4s, v6.4s, v7.4s \n" 189 /* rounds 36-39 */ 190 "sha256su0 v5.4s, v6.4s \n" 191 "mov v18.16b, v0.16b \n" 192 "ld1 {v19.4s}, [x8], #16 \n" 193 "add v16.4s, v6.4s, v19.4s \n" 194 "sha256h q0, q1, v17.4s \n" 195 "sha256h2 q1, q18, v17.4s \n" 196 "sha256su1 v5.4s, v7.4s, v4.4s \n" 197 /* rounds 40-43 */ 198 "sha256su0 v6.4s, v7.4s \n" 199 "mov v18.16b, v0.16b \n" 200 "ld1 {v19.4s}, [x8], #16 \n" 201 "add v17.4s, v7.4s, v19.4s \n" 202 "sha256h q0, q1, v16.4s \n" 203 "sha256h2 q1, q18, v16.4s \n" 204 "sha256su1 v6.4s, v4.4s, v5.4s \n" 205 /* rounds 44-47 */ 206 "sha256su0 v7.4s, v4.4s \n" 207 "mov v18.16b, v0.16b \n" 208 "ld1 {v19.4s}, [x8], #16 \n" 209 "add v16.4s, v4.4s, v19.4s \n" 210 "sha256h q0, q1, v17.4s \n" 211 "sha256h2 q1, q18, v17.4s \n" 212 "sha256su1 v7.4s, v5.4s, v6.4s \n" 213 /* rounds 48-51 (schedule complete) */ 214 "mov v18.16b, v0.16b \n" 215 "ld1 {v19.4s}, [x8], #16 \n" 216 "add v17.4s, v5.4s, v19.4s \n" 217 "sha256h q0, q1, v16.4s \n" 218 "sha256h2 q1, q18, v16.4s \n" 219 /* rounds 52-55 */ 220 "mov v18.16b, v0.16b \n" 221 "ld1 {v19.4s}, [x8], #16 \n" 222 "add v16.4s, v6.4s, v19.4s \n" 223 "sha256h q0, q1, v17.4s \n" 224 "sha256h2 q1, q18, v17.4s \n" 225 /* rounds 56-59 */ 226 "mov v18.16b, v0.16b \n" 227 "ld1 {v19.4s}, [x8], #16 \n" 228 "add v17.4s, v7.4s, v19.4s \n" 229 "sha256h q0, q1, v16.4s \n" 230 "sha256h2 q1, q18, v16.4s \n" 231 /* rounds 60-63 */ 232 "mov v18.16b, v0.16b \n" 233 "sha256h q0, q1, v17.4s \n" 234 "sha256h2 q1, q18, v17.4s \n" 235 /* fold block result back into running state */ 236 "add v0.4s, v0.4s, v2.4s \n" 237 "add v1.4s, v1.4s, v3.4s \n" 238 "subs %[n], %[n], #1 \n" 239 "b.ne 1b \n" 240 "st1 {v0.4s, v1.4s}, [%[st]] \n" 241 : [d] "+r"(p), [n] "+r"(nblocks) 242 : [st] "r"(s->h), [k] "r"(SHA256_K) 243 : "x8", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", 244 "v18", "v19", "memory", "cc"); 245 } 246 #pragma GCC diagnostic pop 247 #else 248 static void sha256_blocks(Sha256* s, const u8* p, size_t nblocks) { 249 while (nblocks--) { 250 sha256_block(s, p); 251 p += 64; 252 } 253 } 254 #endif 255 256 void sha256_update(Sha256* s, const u8* data, u32 n) { 257 s->total += n; 258 /* Top off a partially filled block first. */ 259 if (s->buflen) { 260 u32 take = 64u - s->buflen; 261 if (take > n) take = n; 262 memcpy(s->buf + s->buflen, data, take); 263 s->buflen += take; 264 data += take; 265 n -= take; 266 if (s->buflen == 64) { 267 sha256_blocks(s, s->buf, 1); 268 s->buflen = 0; 269 } 270 } 271 /* Bulk: absorb full blocks straight from the caller's buffer -- no staging 272 * memcpy, which on a multi-MB hash (e.g. the Mach-O code signature) is a 273 * whole extra pass over the data avoided. */ 274 if (n >= 64u) { 275 size_t nb = n >> 6; 276 sha256_blocks(s, data, nb); 277 data += nb * 64u; 278 n -= (u32)(nb * 64u); 279 } 280 /* Stash the sub-block tail. */ 281 if (n) { 282 memcpy(s->buf + s->buflen, data, n); 283 s->buflen += n; 284 } 285 } 286 287 void sha256_final(Sha256* s, u8 out[SHA256_DIGEST_LEN]) { 288 u64 bits = s->total * 8u; 289 u8 pad1 = 0x80; 290 sha256_update(s, &pad1, 1); 291 while (s->buflen != 56) { 292 u8 z = 0; 293 sha256_update(s, &z, 1); 294 } 295 u8 lenbe[8]; 296 for (u32 i = 0; i < 8; ++i) lenbe[7 - i] = (u8)(bits >> (i * 8)); 297 sha256_update(s, lenbe, 8); 298 for (u32 i = 0; i < 8; ++i) { 299 out[i * 4 + 0] = (u8)(s->h[i] >> 24); 300 out[i * 4 + 1] = (u8)(s->h[i] >> 16); 301 out[i * 4 + 2] = (u8)(s->h[i] >> 8); 302 out[i * 4 + 3] = (u8)(s->h[i]); 303 } 304 }