1 //===- AArch64ErrataFix.cpp -----------------------------------------------===//
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
9 // This file implements Section Patching for the purpose of working around
10 // errata in CPUs. The general principle is that an erratum sequence of one or
11 // more instructions is detected in the instruction stream, one of the
12 // instructions in the sequence is replaced with a branch to a patch sequence
13 // of replacement instructions. At the end of the replacement sequence the
14 // patch branches back to the instruction stream.
16 // This technique is only suitable for fixing an erratum when:
17 // - There is a set of necessary conditions required to trigger the erratum that
18 // can be detected at static link time.
19 // - There is a set of replacement instructions that can be used to remove at
20 // least one of the necessary conditions that trigger the erratum.
21 // - We can overwrite an instruction in the erratum sequence with a branch to
22 // the replacement sequence.
23 // - We can place the replacement sequence within range of the branch.
26 // - The implementation here only supports one patch, the AArch64 Cortex-53
27 // errata 843419 that affects r0p0, r0p1, r0p2 and r0p4 versions of the core.
28 // To keep the initial version simple there is no support for multiple
29 // architectures or selection of different patches.
30 //===----------------------------------------------------------------------===//
32 #include "AArch64ErrataFix.h"
34 #include "LinkerScript.h"
35 #include "OutputSections.h"
36 #include "Relocations.h"
39 #include "SyntheticSections.h"
41 #include "lld/Common/Memory.h"
43 #include "llvm/Support/Endian.h"
44 #include "llvm/Support/raw_ostream.h"
48 using namespace llvm::ELF;
49 using namespace llvm::object;
50 using namespace llvm::support;
51 using namespace llvm::support::endian;
54 using namespace lld::elf;
56 // Helper functions to identify instructions and conditions needed to trigger
57 // the Cortex-A53-843419 erratum.
60 // | 1 | immlo (2) | 1 | 0 0 0 0 | immhi (19) | Rd (5) |
61 static bool isADRP(uint32_t Instr) {
62 return (Instr & 0x9f000000) == 0x90000000;
65 // Load and store bit patterns from ARMv8-A ARM ARM.
66 // Instructions appear in order of appearance starting from table in
67 // C4.1.3 Loads and Stores.
69 // All loads and stores have 1 (at bit postion 27), (0 at bit position 25).
70 // | op0 x op1 (2) | 1 op2 0 op3 (2) | x | op4 (5) | xxxx | op5 (2) | x (10) |
71 static bool isLoadStoreClass(uint32_t Instr) {
72 return (Instr & 0x0a000000) == 0x08000000;
75 // LDN/STN multiple no offset
76 // | 0 Q 00 | 1100 | 0 L 00 | 0000 | opcode (4) | size (2) | Rn (5) | Rt (5) |
77 // LDN/STN multiple post-indexed
78 // | 0 Q 00 | 1100 | 1 L 0 | Rm (5)| opcode (4) | size (2) | Rn (5) | Rt (5) |
81 // Utility routine to decode opcode field of LDN/STN multiple structure
82 // instructions to find the ST1 instructions.
83 // opcode == 0010 ST1 4 registers.
84 // opcode == 0110 ST1 3 registers.
85 // opcode == 0111 ST1 1 register.
86 // opcode == 1010 ST1 2 registers.
87 static bool isST1MultipleOpcode(uint32_t Instr) {
88 return (Instr & 0x0000f000) == 0x00002000 ||
89 (Instr & 0x0000f000) == 0x00006000 ||
90 (Instr & 0x0000f000) == 0x00007000 ||
91 (Instr & 0x0000f000) == 0x0000a000;
94 static bool isST1Multiple(uint32_t Instr) {
95 return (Instr & 0xbfff0000) == 0x0c000000 && isST1MultipleOpcode(Instr);
98 // Writes to Rn (writeback).
99 static bool isST1MultiplePost(uint32_t Instr) {
100 return (Instr & 0xbfe00000) == 0x0c800000 && isST1MultipleOpcode(Instr);
103 // LDN/STN single no offset
104 // | 0 Q 00 | 1101 | 0 L R 0 | 0000 | opc (3) S | size (2) | Rn (5) | Rt (5)|
105 // LDN/STN single post-indexed
106 // | 0 Q 00 | 1101 | 1 L R | Rm (5) | opc (3) S | size (2) | Rn (5) | Rt (5)|
109 // Utility routine to decode opcode field of LDN/STN single structure
110 // instructions to find the ST1 instructions.
111 // R == 0 for ST1 and ST3, R == 1 for ST2 and ST4.
112 // opcode == 000 ST1 8-bit.
113 // opcode == 010 ST1 16-bit.
114 // opcode == 100 ST1 32 or 64-bit (Size determines which).
115 static bool isST1SingleOpcode(uint32_t Instr) {
116 return (Instr & 0x0040e000) == 0x00000000 ||
117 (Instr & 0x0040e000) == 0x00004000 ||
118 (Instr & 0x0040e000) == 0x00008000;
121 static bool isST1Single(uint32_t Instr) {
122 return (Instr & 0xbfff0000) == 0x0d000000 && isST1SingleOpcode(Instr);
125 // Writes to Rn (writeback).
126 static bool isST1SinglePost(uint32_t Instr) {
127 return (Instr & 0xbfe00000) == 0x0d800000 && isST1SingleOpcode(Instr);
130 static bool isST1(uint32_t Instr) {
131 return isST1Multiple(Instr) || isST1MultiplePost(Instr) ||
132 isST1Single(Instr) || isST1SinglePost(Instr);
135 // Load/store exclusive
136 // | size (2) 00 | 1000 | o2 L o1 | Rs (5) | o0 | Rt2 (5) | Rn (5) | Rt (5) |
137 // L == 0 for Stores.
138 static bool isLoadStoreExclusive(uint32_t Instr) {
139 return (Instr & 0x3f000000) == 0x08000000;
142 static bool isLoadExclusive(uint32_t Instr) {
143 return (Instr & 0x3f400000) == 0x08400000;
146 // Load register literal
147 // | opc (2) 01 | 1 V 00 | imm19 | Rt (5) |
148 static bool isLoadLiteral(uint32_t Instr) {
149 return (Instr & 0x3b000000) == 0x18000000;
152 // Load/store no-allocate pair
154 // | opc (2) 10 | 1 V 00 | 0 L | imm7 | Rt2 (5) | Rn (5) | Rt (5) |
155 // L == 0 for stores.
156 // Never writes to register
157 static bool isSTNP(uint32_t Instr) {
158 return (Instr & 0x3bc00000) == 0x28000000;
161 // Load/store register pair
163 // | opc (2) 10 | 1 V 00 | 1 L | imm7 | Rt2 (5) | Rn (5) | Rt (5) |
164 // L == 0 for stores, V == 0 for Scalar, V == 1 for Simd/FP
166 static bool isSTPPost(uint32_t Instr) {
167 return (Instr & 0x3bc00000) == 0x28800000;
171 // | opc (2) 10 | 1 V 01 | 0 L | imm7 | Rt2 (5) | Rn (5) | Rt (5) |
172 static bool isSTPOffset(uint32_t Instr) {
173 return (Instr & 0x3bc00000) == 0x29000000;
177 // | opc (2) 10 | 1 V 01 | 1 L | imm7 | Rt2 (5) | Rn (5) | Rt (5) |
179 static bool isSTPPre(uint32_t Instr) {
180 return (Instr & 0x3bc00000) == 0x29800000;
183 static bool isSTP(uint32_t Instr) {
184 return isSTPPost(Instr) || isSTPOffset(Instr) || isSTPPre(Instr);
187 // Load/store register (unscaled immediate)
188 // | size (2) 11 | 1 V 00 | opc (2) 0 | imm9 | 00 | Rn (5) | Rt (5) |
189 // V == 0 for Scalar, V == 1 for Simd/FP.
190 static bool isLoadStoreUnscaled(uint32_t Instr) {
191 return (Instr & 0x3b000c00) == 0x38000000;
194 // Load/store register (immediate post-indexed)
195 // | size (2) 11 | 1 V 00 | opc (2) 0 | imm9 | 01 | Rn (5) | Rt (5) |
196 static bool isLoadStoreImmediatePost(uint32_t Instr) {
197 return (Instr & 0x3b200c00) == 0x38000400;
200 // Load/store register (unprivileged)
201 // | size (2) 11 | 1 V 00 | opc (2) 0 | imm9 | 10 | Rn (5) | Rt (5) |
202 static bool isLoadStoreUnpriv(uint32_t Instr) {
203 return (Instr & 0x3b200c00) == 0x38000800;
206 // Load/store register (immediate pre-indexed)
207 // | size (2) 11 | 1 V 00 | opc (2) 0 | imm9 | 11 | Rn (5) | Rt (5) |
208 static bool isLoadStoreImmediatePre(uint32_t Instr) {
209 return (Instr & 0x3b200c00) == 0x38000c00;
212 // Load/store register (register offset)
213 // | size (2) 11 | 1 V 00 | opc (2) 1 | Rm (5) | option (3) S | 10 | Rn | Rt |
214 static bool isLoadStoreRegisterOff(uint32_t Instr) {
215 return (Instr & 0x3b200c00) == 0x38200800;
218 // Load/store register (unsigned immediate)
219 // | size (2) 11 | 1 V 01 | opc (2) | imm12 | Rn (5) | Rt (5) |
220 static bool isLoadStoreRegisterUnsigned(uint32_t Instr) {
221 return (Instr & 0x3b000000) == 0x39000000;
224 // Rt is always in bit position 0 - 4.
225 static uint32_t getRt(uint32_t Instr) { return (Instr & 0x1f); }
227 // Rn is always in bit position 5 - 9.
228 static uint32_t getRn(uint32_t Instr) { return (Instr >> 5) & 0x1f; }
230 // C4.1.2 Branches, Exception Generating and System instructions
231 // | op0 (3) 1 | 01 op1 (4) | x (22) |
232 // op0 == 010 101 op1 == 0xxx Conditional Branch.
233 // op0 == 110 101 op1 == 1xxx Unconditional Branch Register.
234 // op0 == x00 101 op1 == xxxx Unconditional Branch immediate.
235 // op0 == x01 101 op1 == 0xxx Compare and branch immediate.
236 // op0 == x01 101 op1 == 1xxx Test and branch immediate.
237 static bool isBranch(uint32_t Instr) {
238 return ((Instr & 0xfe000000) == 0xd6000000) || // Cond branch.
239 ((Instr & 0xfe000000) == 0x54000000) || // Uncond branch reg.
240 ((Instr & 0x7c000000) == 0x14000000) || // Uncond branch imm.
241 ((Instr & 0x7c000000) == 0x34000000); // Compare and test branch.
244 static bool isV8SingleRegisterNonStructureLoadStore(uint32_t Instr) {
245 return isLoadStoreUnscaled(Instr) || isLoadStoreImmediatePost(Instr) ||
246 isLoadStoreUnpriv(Instr) || isLoadStoreImmediatePre(Instr) ||
247 isLoadStoreRegisterOff(Instr) || isLoadStoreRegisterUnsigned(Instr);
250 // Note that this function refers to v8.0 only and does not include the
251 // additional load and store instructions added for in later revisions of
252 // the architecture such as the Atomic memory operations introduced
254 static bool isV8NonStructureLoad(uint32_t Instr) {
255 if (isLoadExclusive(Instr))
257 if (isLoadLiteral(Instr))
259 else if (isV8SingleRegisterNonStructureLoadStore(Instr)) {
260 // For Load and Store single register, Loads are derived from a
261 // combination of the Size, V and Opc fields.
262 uint32_t Size = (Instr >> 30) & 0xff;
263 uint32_t V = (Instr >> 26) & 0x1;
264 uint32_t Opc = (Instr >> 22) & 0x3;
265 // For the load and store instructions that we are decoding.
266 // Opc == 0 are all stores.
267 // Opc == 1 with a couple of exceptions are loads. The exceptions are:
268 // Size == 00 (0), V == 1, Opc == 10 (2) which is a store and
269 // Size == 11 (3), V == 0, Opc == 10 (2) which is a prefetch.
270 return Opc != 0 && !(Size == 0 && V == 1 && Opc == 2) &&
271 !(Size == 3 && V == 0 && Opc == 2);
276 // The following decode instructions are only complete up to the instructions
277 // needed for errata 843419.
279 // Instruction with writeback updates the index register after the load/store.
280 static bool hasWriteback(uint32_t Instr) {
281 return isLoadStoreImmediatePre(Instr) || isLoadStoreImmediatePost(Instr) ||
282 isSTPPre(Instr) || isSTPPost(Instr) || isST1SinglePost(Instr) ||
283 isST1MultiplePost(Instr);
286 // For the load and store class of instructions, a load can write to the
287 // destination register, a load and a store can write to the base register when
288 // the instruction has writeback.
289 static bool doesLoadStoreWriteToReg(uint32_t Instr, uint32_t Reg) {
290 return (isV8NonStructureLoad(Instr) && getRt(Instr) == Reg) ||
291 (hasWriteback(Instr) && getRn(Instr) == Reg);
294 // Scanner for Cortex-A53 errata 843419
295 // Full details are available in the Cortex A53 MPCore revision 0 Software
296 // Developers Errata Notice (ARM-EPM-048406).
298 // The instruction sequence that triggers the erratum is common in compiled
299 // AArch64 code, however it is sensitive to the offset of the sequence within
300 // a 4k page. This means that by scanning and fixing the patch after we have
301 // assigned addresses we only need to disassemble and fix instances of the
302 // sequence in the range of affected offsets.
304 // In summary the erratum conditions are a series of 4 instructions:
305 // 1.) An ADRP instruction that writes to register Rn with low 12 bits of
306 // address of instruction either 0xff8 or 0xffc.
307 // 2.) A load or store instruction that can be:
308 // - A single register load or store, of either integer or vector registers.
309 // - An STP or STNP, of either integer or vector registers.
310 // - An Advanced SIMD ST1 store instruction.
311 // - Must not write to Rn, but may optionally read from it.
312 // 3.) An optional instruction that is not a branch and does not write to Rn.
313 // 4.) A load or store from the Load/store register (unsigned immediate) class
314 // that uses Rn as the base address register.
316 // Note that we do not attempt to scan for Sequence 2 as described in the
317 // Software Developers Errata Notice as this has been assessed to be extremely
318 // unlikely to occur in compiled code. This matches gold and ld.bfd behavior.
320 // Return true if the Instruction sequence Adrp, Instr2, and Instr4 match
321 // the erratum sequence. The Adrp, Instr2 and Instr4 correspond to 1.), 2.),
322 // and 4.) in the Scanner for Cortex-A53 errata comment above.
323 static bool is843419ErratumSequence(uint32_t Instr1, uint32_t Instr2,
328 uint32_t Rn = getRt(Instr1);
329 return isLoadStoreClass(Instr2) &&
330 (isLoadStoreExclusive(Instr2) || isLoadLiteral(Instr2) ||
331 isV8SingleRegisterNonStructureLoadStore(Instr2) || isSTP(Instr2) ||
332 isSTNP(Instr2) || isST1(Instr2)) &&
333 !doesLoadStoreWriteToReg(Instr2, Rn) &&
334 isLoadStoreRegisterUnsigned(Instr4) && getRn(Instr4) == Rn;
337 // Scan the instruction sequence starting at Offset Off from the base of
338 // InputSection IS. We update Off in this function rather than in the caller as
339 // we can skip ahead much further into the section when we know how many
340 // instructions we've scanned.
341 // Return the offset of the load or store instruction in IS that we want to
342 // patch or 0 if no patch required.
343 static uint64_t scanCortexA53Errata843419(InputSection *IS, uint64_t &Off,
345 uint64_t ISAddr = IS->getParent()->Addr + IS->OutSecOff;
347 // Advance Off so that (ISAddr + Off) modulo 0x1000 is at least 0xff8.
348 uint64_t InitialPageOff = (ISAddr + Off) & 0xfff;
349 if (InitialPageOff < 0xff8)
350 Off += 0xff8 - InitialPageOff;
352 bool OptionalAllowed = Limit - Off > 12;
353 if (Off >= Limit || Limit - Off < 12) {
354 // Need at least 3 4-byte sized instructions to trigger erratum.
359 uint64_t PatchOff = 0;
360 const uint8_t *Buf = IS->Data.begin();
361 const ulittle32_t *InstBuf = reinterpret_cast<const ulittle32_t *>(Buf + Off);
362 uint32_t Instr1 = *InstBuf++;
363 uint32_t Instr2 = *InstBuf++;
364 uint32_t Instr3 = *InstBuf++;
365 if (is843419ErratumSequence(Instr1, Instr2, Instr3)) {
367 } else if (OptionalAllowed && !isBranch(Instr3)) {
368 uint32_t Instr4 = *InstBuf++;
369 if (is843419ErratumSequence(Instr1, Instr2, Instr4))
372 if (((ISAddr + Off) & 0xfff) == 0xff8)
379 class lld::elf::Patch843419Section : public SyntheticSection {
381 Patch843419Section(InputSection *P, uint64_t Off);
383 void writeTo(uint8_t *Buf) override;
385 size_t getSize() const override { return 8; }
387 uint64_t getLDSTAddr() const;
389 // The Section we are patching.
390 const InputSection *Patchee;
391 // The offset of the instruction in the Patchee section we are patching.
392 uint64_t PatcheeOffset;
393 // A label for the start of the Patch that we can use as a relocation target.
397 lld::elf::Patch843419Section::Patch843419Section(InputSection *P, uint64_t Off)
398 : SyntheticSection(SHF_ALLOC | SHF_EXECINSTR, SHT_PROGBITS, 4,
400 Patchee(P), PatcheeOffset(Off) {
401 this->Parent = P->getParent();
402 PatchSym = addSyntheticLocal(
403 Saver.save("__CortexA53843419_" + utohexstr(getLDSTAddr())), STT_FUNC, 0,
405 addSyntheticLocal(Saver.save("$x"), STT_NOTYPE, 0, 0, *this);
408 uint64_t lld::elf::Patch843419Section::getLDSTAddr() const {
409 return Patchee->getParent()->Addr + Patchee->OutSecOff + PatcheeOffset;
412 void lld::elf::Patch843419Section::writeTo(uint8_t *Buf) {
413 // Copy the instruction that we will be replacing with a branch in the
415 write32le(Buf, read32le(Patchee->Data.begin() + PatcheeOffset));
417 // Apply any relocation transferred from the original PatcheeSection.
418 // For a SyntheticSection Buf already has OutSecOff added, but relocateAlloc
419 // also adds OutSecOff so we need to subtract to avoid double counting.
420 this->relocateAlloc(Buf - OutSecOff, Buf - OutSecOff + getSize());
422 // Return address is the next instruction after the one we have just copied.
423 uint64_t S = getLDSTAddr() + 4;
424 uint64_t P = PatchSym->getVA() + 4;
425 Target->relocateOne(Buf + 4, R_AARCH64_JUMP26, S - P);
428 void AArch64Err843419Patcher::init() {
429 // The AArch64 ABI permits data in executable sections. We must avoid scanning
430 // this data as if it were instructions to avoid false matches. We use the
431 // mapping symbols in the InputObjects to identify this data, caching the
432 // results in SectionMap so we don't have to recalculate it each pass.
434 // The ABI Section 4.5.4 Mapping symbols; defines local symbols that describe
435 // half open intervals [Symbol Value, Next Symbol Value) of code and data
436 // within sections. If there is no next symbol then the half open interval is
437 // [Symbol Value, End of section). The type, code or data, is determined by
438 // the mapping symbol name, $x for code, $d for data.
439 auto IsCodeMapSymbol = [](const Symbol *B) {
440 return B->getName() == "$x" || B->getName().startswith("$x.");
442 auto IsDataMapSymbol = [](const Symbol *B) {
443 return B->getName() == "$d" || B->getName().startswith("$d.");
446 // Collect mapping symbols for every executable InputSection.
447 for (InputFile *File : ObjectFiles) {
448 auto *F = cast<ObjFile<ELF64LE>>(File);
449 for (Symbol *B : F->getLocalSymbols()) {
450 auto *Def = dyn_cast<Defined>(B);
453 if (!IsCodeMapSymbol(Def) && !IsDataMapSymbol(Def))
455 if (auto *Sec = dyn_cast<InputSection>(Def->Section))
456 if (Sec->Flags & SHF_EXECINSTR)
457 SectionMap[Sec].push_back(Def);
460 // For each InputSection make sure the mapping symbols are in sorted in
461 // ascending order and free from consecutive runs of mapping symbols with
462 // the same type. For example we must remove the redundant $d.1 from $x.0
464 for (auto &KV : SectionMap) {
465 std::vector<const Defined *> &MapSyms = KV.second;
466 if (MapSyms.size() <= 1)
469 MapSyms.begin(), MapSyms.end(),
470 [](const Defined *A, const Defined *B) { return A->Value < B->Value; });
472 std::unique(MapSyms.begin(), MapSyms.end(),
473 [=](const Defined *A, const Defined *B) {
474 return (IsCodeMapSymbol(A) && IsCodeMapSymbol(B)) ||
475 (IsDataMapSymbol(A) && IsDataMapSymbol(B));
482 // Insert the PatchSections we have created back into the
483 // InputSectionDescription. As inserting patches alters the addresses of
484 // InputSections that follow them, we try and place the patches after all the
485 // executable sections, although we may need to insert them earlier if the
486 // InputSectionDescription is larger than the maximum branch range.
487 void AArch64Err843419Patcher::insertPatches(
488 InputSectionDescription &ISD, std::vector<Patch843419Section *> &Patches) {
490 uint64_t PrevISLimit = ISD.Sections.front()->OutSecOff;
491 uint64_t PatchUpperBound = PrevISLimit + Target->ThunkSectionSpacing;
493 // Set the OutSecOff of patches to the place where we want to insert them.
494 // We use a similar strategy to Thunk placement. Place patches roughly
495 // every multiple of maximum branch range.
496 auto PatchIt = Patches.begin();
497 auto PatchEnd = Patches.end();
498 for (const InputSection *IS : ISD.Sections) {
499 ISLimit = IS->OutSecOff + IS->getSize();
500 if (ISLimit > PatchUpperBound) {
501 while (PatchIt != PatchEnd) {
502 if ((*PatchIt)->getLDSTAddr() >= PrevISLimit)
504 (*PatchIt)->OutSecOff = PrevISLimit;
507 PatchUpperBound = PrevISLimit + Target->ThunkSectionSpacing;
509 PrevISLimit = ISLimit;
511 for (; PatchIt != PatchEnd; ++PatchIt) {
512 (*PatchIt)->OutSecOff = ISLimit;
515 // merge all patch sections. We use the OutSecOff assigned above to
516 // determine the insertion point. This is ok as we only merge into an
517 // InputSectionDescription once per pass, and at the end of the pass
518 // assignAddresses() will recalculate all the OutSecOff values.
519 std::vector<InputSection *> Tmp;
520 Tmp.reserve(ISD.Sections.size() + Patches.size());
521 auto MergeCmp = [](const InputSection *A, const InputSection *B) {
522 if (A->OutSecOff < B->OutSecOff)
524 if (A->OutSecOff == B->OutSecOff && isa<Patch843419Section>(A) &&
525 !isa<Patch843419Section>(B))
529 std::merge(ISD.Sections.begin(), ISD.Sections.end(), Patches.begin(),
530 Patches.end(), std::back_inserter(Tmp), MergeCmp);
531 ISD.Sections = std::move(Tmp);
534 // Given an erratum sequence that starts at address AdrpAddr, with an
535 // instruction that we need to patch at PatcheeOffset from the start of
536 // InputSection IS, create a Patch843419 Section and add it to the
537 // Patches that we need to insert.
538 static void implementPatch(uint64_t AdrpAddr, uint64_t PatcheeOffset,
540 std::vector<Patch843419Section *> &Patches) {
541 // There may be a relocation at the same offset that we are patching. There
542 // are three cases that we need to consider.
543 // Case 1: R_AARCH64_JUMP26 branch relocation. We have already patched this
544 // instance of the erratum on a previous patch and altered the relocation. We
545 // have nothing more to do.
546 // Case 2: A load/store register (unsigned immediate) class relocation. There
547 // are two of these R_AARCH_LD64_ABS_LO12_NC and R_AARCH_LD64_GOT_LO12_NC and
548 // they are both absolute. We need to add the same relocation to the patch,
549 // and replace the relocation with a R_AARCH_JUMP26 branch relocation.
550 // Case 3: No relocation. We must create a new R_AARCH64_JUMP26 branch
551 // relocation at the offset.
552 auto RelIt = std::find_if(
553 IS->Relocations.begin(), IS->Relocations.end(),
554 [=](const Relocation &R) { return R.Offset == PatcheeOffset; });
555 if (RelIt != IS->Relocations.end() && RelIt->Type == R_AARCH64_JUMP26)
559 message("detected cortex-a53-843419 erratum sequence starting at " +
560 utohexstr(AdrpAddr) + " in unpatched output.");
562 auto *PS = make<Patch843419Section>(IS, PatcheeOffset);
563 Patches.push_back(PS);
565 auto MakeRelToPatch = [](uint64_t Offset, Symbol *PatchSym) {
566 return Relocation{R_PC, R_AARCH64_JUMP26, Offset, 0, PatchSym};
569 if (RelIt != IS->Relocations.end()) {
570 PS->Relocations.push_back(
571 {RelIt->Expr, RelIt->Type, 0, RelIt->Addend, RelIt->Sym});
572 *RelIt = MakeRelToPatch(PatcheeOffset, PS->PatchSym);
574 IS->Relocations.push_back(MakeRelToPatch(PatcheeOffset, PS->PatchSym));
577 // Scan all the instructions in InputSectionDescription, for each instance of
578 // the erratum sequence create a Patch843419Section. We return the list of
579 // Patch843419Sections that need to be applied to ISD.
580 std::vector<Patch843419Section *>
581 AArch64Err843419Patcher::patchInputSectionDescription(
582 InputSectionDescription &ISD) {
583 std::vector<Patch843419Section *> Patches;
584 for (InputSection *IS : ISD.Sections) {
585 // LLD doesn't use the erratum sequence in SyntheticSections.
586 if (isa<SyntheticSection>(IS))
588 // Use SectionMap to make sure we only scan code and not inline data.
589 // We have already sorted MapSyms in ascending order and removed consecutive
590 // mapping symbols of the same type. Our range of executable instructions to
591 // scan is therefore [CodeSym->Value, DataSym->Value) or [CodeSym->Value,
593 std::vector<const Defined *> &MapSyms = SectionMap[IS];
595 auto CodeSym = llvm::find_if(MapSyms, [&](const Defined *MS) {
596 return MS->getName().startswith("$x");
599 while (CodeSym != MapSyms.end()) {
600 auto DataSym = std::next(CodeSym);
601 uint64_t Off = (*CodeSym)->Value;
603 (DataSym == MapSyms.end()) ? IS->Data.size() : (*DataSym)->Value;
605 while (Off < Limit) {
606 uint64_t StartAddr = IS->getParent()->Addr + IS->OutSecOff + Off;
607 if (uint64_t PatcheeOffset = scanCortexA53Errata843419(IS, Off, Limit))
608 implementPatch(StartAddr, PatcheeOffset, IS, Patches);
610 if (DataSym == MapSyms.end())
612 CodeSym = std::next(DataSym);
618 // For each InputSectionDescription make one pass over the executable sections
619 // looking for the erratum sequence; creating a synthetic Patch843419Section
620 // for each instance found. We insert these synthetic patch sections after the
621 // executable code in each InputSectionDescription.
624 // The Output and Input Sections have had their final addresses assigned.
627 // Returns true if at least one patch was added. The addresses of the
628 // Ouptut and Input Sections may have been changed.
629 // Returns false if no patches were required and no changes were made.
630 bool AArch64Err843419Patcher::createFixes() {
631 if (Initialized == false)
634 bool AddressesChanged = false;
635 for (OutputSection *OS : OutputSections) {
636 if (!(OS->Flags & SHF_ALLOC) || !(OS->Flags & SHF_EXECINSTR))
638 for (BaseCommand *BC : OS->SectionCommands)
639 if (auto *ISD = dyn_cast<InputSectionDescription>(BC)) {
640 std::vector<Patch843419Section *> Patches =
641 patchInputSectionDescription(*ISD);
642 if (!Patches.empty()) {
643 insertPatches(*ISD, Patches);
644 AddressesChanged = true;
648 return AddressesChanged;