cddl/contrib/opensolaris/lib/libdtrace/common/dt_link.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26
  27 #pragma ident   "%Z%%M% %I%     %E% SMI"
  28
  29 #define ELF_TARGET_ALL
  30 #include <elf.h>
  31
  32 #include <sys/types.h>
  33 #if defined(sun)
  34 #include <sys/sysmacros.h>
  35 #else
  36 #define P2ROUNDUP(x, align)             (-(-(x) & -(align)))
  37 #endif
  38
  39 #include <unistd.h>
  40 #include <strings.h>
  41 #if defined(sun)
  42 #include <alloca.h>
  43 #endif
  44 #include <limits.h>
  45 #include <stddef.h>
  46 #include <stdlib.h>
  47 #include <stdio.h>
  48 #include <fcntl.h>
  49 #include <errno.h>
  50 #if defined(sun)
  51 #include <wait.h>
  52 #else
  53 #include <sys/wait.h>
  54 #include <libelf.h>
  55 #include <gelf.h>
  56 #include <sys/mman.h>
  57 #endif
  58 #include <assert.h>
  59 #include <sys/ipc.h>
  60
  61 #include <dt_impl.h>
  62 #include <dt_provider.h>
  63 #include <dt_program.h>
  64 #include <dt_string.h>
  65
  66 #define ESHDR_NULL      0
  67 #define ESHDR_SHSTRTAB  1
  68 #define ESHDR_DOF       2
  69 #define ESHDR_STRTAB    3
  70 #define ESHDR_SYMTAB    4
  71 #define ESHDR_REL       5
  72 #define ESHDR_NUM       6
  73
  74 #define PWRITE_SCN(index, data) \
  75         (lseek64(fd, (off64_t)elf_file.shdr[(index)].sh_offset, SEEK_SET) != \
  76         (off64_t)elf_file.shdr[(index)].sh_offset || \
  77         dt_write(dtp, fd, (data), elf_file.shdr[(index)].sh_size) != \
  78         elf_file.shdr[(index)].sh_size)
  79
  80 static const char DTRACE_SHSTRTAB32[] = "\0"
  81 ".shstrtab\0"           /* 1 */
  82 ".SUNW_dof\0"           /* 11 */
  83 ".strtab\0"             /* 21 */
  84 ".symtab\0"             /* 29 */
  85 #ifdef __sparc
  86 ".rela.SUNW_dof";       /* 37 */
  87 #else
  88 ".rel.SUNW_dof";        /* 37 */
  89 #endif
  90
  91 static const char DTRACE_SHSTRTAB64[] = "\0"
  92 ".shstrtab\0"           /* 1 */
  93 ".SUNW_dof\0"           /* 11 */
  94 ".strtab\0"             /* 21 */
  95 ".symtab\0"             /* 29 */
  96 ".rela.SUNW_dof";       /* 37 */
  97
  98 static const char DOFSTR[] = "__SUNW_dof";
  99 static const char DOFLAZYSTR[] = "___SUNW_dof";
 100
 101 typedef struct dt_link_pair {
 102         struct dt_link_pair *dlp_next;  /* next pair in linked list */
 103         void *dlp_str;                  /* buffer for string table */
 104         void *dlp_sym;                  /* buffer for symbol table */
 105 } dt_link_pair_t;
 106
 107 typedef struct dof_elf32 {
 108         uint32_t de_nrel;               /* relocation count */
 109 #ifdef __sparc
 110         Elf32_Rela *de_rel;             /* array of relocations for sparc */
 111 #else
 112         Elf32_Rel *de_rel;              /* array of relocations for x86 */
 113 #endif
 114         uint32_t de_nsym;               /* symbol count */
 115         Elf32_Sym *de_sym;              /* array of symbols */
 116         uint32_t de_strlen;             /* size of of string table */
 117         char *de_strtab;                /* string table */
 118         uint32_t de_global;             /* index of the first global symbol */
 119 } dof_elf32_t;
 120
 121 static int
 122 prepare_elf32(dtrace_hdl_t *dtp, const dof_hdr_t *dof, dof_elf32_t *dep)
 123 {
 124         dof_sec_t *dofs, *s;
 125         dof_relohdr_t *dofrh;
 126         dof_relodesc_t *dofr;
 127         char *strtab;
 128         int i, j, nrel;
 129         size_t strtabsz = 1;
 130         uint32_t count = 0;
 131         size_t base;
 132         Elf32_Sym *sym;
 133 #ifdef __sparc
 134         Elf32_Rela *rel;
 135 #else
 136         Elf32_Rel *rel;
 137 #endif
 138
 139         /*LINTED*/
 140         dofs = (dof_sec_t *)((char *)dof + dof->dofh_secoff);
 141
 142         /*
 143          * First compute the size of the string table and the number of
 144          * relocations present in the DOF.
 145          */
 146         for (i = 0; i < dof->dofh_secnum; i++) {
 147                 if (dofs[i].dofs_type != DOF_SECT_URELHDR)
 148                         continue;
 149
 150                 /*LINTED*/
 151                 dofrh = (dof_relohdr_t *)((char *)dof + dofs[i].dofs_offset);
 152
 153                 s = &dofs[dofrh->dofr_strtab];
 154                 strtab = (char *)dof + s->dofs_offset;
 155                 assert(strtab[0] == '\0');
 156                 strtabsz += s->dofs_size - 1;
 157
 158                 s = &dofs[dofrh->dofr_relsec];
 159                 /*LINTED*/
 160                 dofr = (dof_relodesc_t *)((char *)dof + s->dofs_offset);
 161                 count += s->dofs_size / s->dofs_entsize;
 162         }
 163
 164         dep->de_strlen = strtabsz;
 165         dep->de_nrel = count;
 166         dep->de_nsym = count + 1; /* the first symbol is always null */
 167
 168         if (dtp->dt_lazyload) {
 169                 dep->de_strlen += sizeof (DOFLAZYSTR);
 170                 dep->de_nsym++;
 171         } else {
 172                 dep->de_strlen += sizeof (DOFSTR);
 173                 dep->de_nsym++;
 174         }
 175
 176         if ((dep->de_rel = calloc(dep->de_nrel,
 177             sizeof (dep->de_rel[0]))) == NULL) {
 178                 return (dt_set_errno(dtp, EDT_NOMEM));
 179         }
 180
 181         if ((dep->de_sym = calloc(dep->de_nsym, sizeof (Elf32_Sym))) == NULL) {
 182                 free(dep->de_rel);
 183                 return (dt_set_errno(dtp, EDT_NOMEM));
 184         }
 185
 186         if ((dep->de_strtab = calloc(dep->de_strlen, 1)) == NULL) {
 187                 free(dep->de_rel);
 188                 free(dep->de_sym);
 189                 return (dt_set_errno(dtp, EDT_NOMEM));
 190         }
 191
 192         count = 0;
 193         strtabsz = 1;
 194         dep->de_strtab[0] = '\0';
 195         rel = dep->de_rel;
 196         sym = dep->de_sym;
 197         dep->de_global = 1;
 198
 199         /*
 200          * The first symbol table entry must be zeroed and is always ignored.
 201          */
 202         bzero(sym, sizeof (Elf32_Sym));
 203         sym++;
 204
 205         /*
 206          * Take a second pass through the DOF sections filling in the
 207          * memory we allocated.
 208          */
 209         for (i = 0; i < dof->dofh_secnum; i++) {
 210                 if (dofs[i].dofs_type != DOF_SECT_URELHDR)
 211                         continue;
 212
 213                 /*LINTED*/
 214                 dofrh = (dof_relohdr_t *)((char *)dof + dofs[i].dofs_offset);
 215
 216                 s = &dofs[dofrh->dofr_strtab];
 217                 strtab = (char *)dof + s->dofs_offset;
 218                 bcopy(strtab + 1, dep->de_strtab + strtabsz, s->dofs_size);
 219                 base = strtabsz;
 220                 strtabsz += s->dofs_size - 1;
 221
 222                 s = &dofs[dofrh->dofr_relsec];
 223                 /*LINTED*/
 224                 dofr = (dof_relodesc_t *)((char *)dof + s->dofs_offset);
 225                 nrel = s->dofs_size / s->dofs_entsize;
 226
 227                 s = &dofs[dofrh->dofr_tgtsec];
 228
 229                 for (j = 0; j < nrel; j++) {
 230 #if defined(__arm__)
 231 /* XXX */
 232 printf("%s:%s(%d): DOODAD\n",__FUNCTION__,__FILE__,__LINE__);
 233 #elif defined(__ia64__)
 234 /* XXX */
 235 printf("%s:%s(%d): DOODAD\n",__FUNCTION__,__FILE__,__LINE__);
 236 #elif defined(__i386) || defined(__amd64)
 237                         rel->r_offset = s->dofs_offset +
 238                             dofr[j].dofr_offset;
 239                         rel->r_info = ELF32_R_INFO(count + dep->de_global,
 240                             R_386_32);
 241 #elif defined(__mips__)
 242 /* XXX */
 243 printf("%s:%s(%d): DOODAD\n",__FUNCTION__,__FILE__,__LINE__);
 244 #elif defined(__powerpc__)
 245 /* XXX */
 246 printf("%s:%s(%d): DOODAD\n",__FUNCTION__,__FILE__,__LINE__);
 247 #elif defined(__sparc)
 248                         /*
 249                          * Add 4 bytes to hit the low half of this 64-bit
 250                          * big-endian address.
 251                          */
 252                         rel->r_offset = s->dofs_offset +
 253                             dofr[j].dofr_offset + 4;
 254                         rel->r_info = ELF32_R_INFO(count + dep->de_global,
 255                             R_SPARC_32);
 256 #else
 257 #error unknown ISA
 258 #endif
 259
 260                         sym->st_name = base + dofr[j].dofr_name - 1;
 261                         sym->st_value = 0;
 262                         sym->st_size = 0;
 263                         sym->st_info = ELF32_ST_INFO(STB_GLOBAL, STT_FUNC);
 264                         sym->st_other = 0;
 265                         sym->st_shndx = SHN_UNDEF;
 266
 267                         rel++;
 268                         sym++;
 269                         count++;
 270                 }
 271         }
 272
 273         /*
 274          * Add a symbol for the DOF itself. We use a different symbol for
 275          * lazily and actively loaded DOF to make them easy to distinguish.
 276          */
 277         sym->st_name = strtabsz;
 278         sym->st_value = 0;
 279         sym->st_size = dof->dofh_filesz;
 280         sym->st_info = ELF32_ST_INFO(STB_GLOBAL, STT_OBJECT);
 281         sym->st_other = 0;
 282         sym->st_shndx = ESHDR_DOF;
 283         sym++;
 284
 285         if (dtp->dt_lazyload) {
 286                 bcopy(DOFLAZYSTR, dep->de_strtab + strtabsz,
 287                     sizeof (DOFLAZYSTR));
 288                 strtabsz += sizeof (DOFLAZYSTR);
 289         } else {
 290                 bcopy(DOFSTR, dep->de_strtab + strtabsz, sizeof (DOFSTR));
 291                 strtabsz += sizeof (DOFSTR);
 292         }
 293
 294         assert(count == dep->de_nrel);
 295         assert(strtabsz == dep->de_strlen);
 296
 297         return (0);
 298 }
 299
 300
 301 typedef struct dof_elf64 {
 302         uint32_t de_nrel;
 303         Elf64_Rela *de_rel;
 304         uint32_t de_nsym;
 305         Elf64_Sym *de_sym;
 306
 307         uint32_t de_strlen;
 308         char *de_strtab;
 309
 310         uint32_t de_global;
 311 } dof_elf64_t;
 312
 313 static int
 314 prepare_elf64(dtrace_hdl_t *dtp, const dof_hdr_t *dof, dof_elf64_t *dep)
 315 {
 316         dof_sec_t *dofs, *s;
 317         dof_relohdr_t *dofrh;
 318         dof_relodesc_t *dofr;
 319         char *strtab;
 320         int i, j, nrel;
 321         size_t strtabsz = 1;
 322         uint32_t count = 0;
 323         size_t base;
 324         Elf64_Sym *sym;
 325         Elf64_Rela *rel;
 326
 327         /*LINTED*/
 328         dofs = (dof_sec_t *)((char *)dof + dof->dofh_secoff);
 329
 330         /*
 331          * First compute the size of the string table and the number of
 332          * relocations present in the DOF.
 333          */
 334         for (i = 0; i < dof->dofh_secnum; i++) {
 335                 if (dofs[i].dofs_type != DOF_SECT_URELHDR)
 336                         continue;
 337
 338                 /*LINTED*/
 339                 dofrh = (dof_relohdr_t *)((char *)dof + dofs[i].dofs_offset);
 340
 341                 s = &dofs[dofrh->dofr_strtab];
 342                 strtab = (char *)dof + s->dofs_offset;
 343                 assert(strtab[0] == '\0');
 344                 strtabsz += s->dofs_size - 1;
 345
 346                 s = &dofs[dofrh->dofr_relsec];
 347                 /*LINTED*/
 348                 dofr = (dof_relodesc_t *)((char *)dof + s->dofs_offset);
 349                 count += s->dofs_size / s->dofs_entsize;
 350         }
 351
 352         dep->de_strlen = strtabsz;
 353         dep->de_nrel = count;
 354         dep->de_nsym = count + 1; /* the first symbol is always null */
 355
 356         if (dtp->dt_lazyload) {
 357                 dep->de_strlen += sizeof (DOFLAZYSTR);
 358                 dep->de_nsym++;
 359         } else {
 360                 dep->de_strlen += sizeof (DOFSTR);
 361                 dep->de_nsym++;
 362         }
 363
 364         if ((dep->de_rel = calloc(dep->de_nrel,
 365             sizeof (dep->de_rel[0]))) == NULL) {
 366                 return (dt_set_errno(dtp, EDT_NOMEM));
 367         }
 368
 369         if ((dep->de_sym = calloc(dep->de_nsym, sizeof (Elf64_Sym))) == NULL) {
 370                 free(dep->de_rel);
 371                 return (dt_set_errno(dtp, EDT_NOMEM));
 372         }
 373
 374         if ((dep->de_strtab = calloc(dep->de_strlen, 1)) == NULL) {
 375                 free(dep->de_rel);
 376                 free(dep->de_sym);
 377                 return (dt_set_errno(dtp, EDT_NOMEM));
 378         }
 379
 380         count = 0;
 381         strtabsz = 1;
 382         dep->de_strtab[0] = '\0';
 383         rel = dep->de_rel;
 384         sym = dep->de_sym;
 385         dep->de_global = 1;
 386
 387         /*
 388          * The first symbol table entry must be zeroed and is always ignored.
 389          */
 390         bzero(sym, sizeof (Elf64_Sym));
 391         sym++;
 392
 393         /*
 394          * Take a second pass through the DOF sections filling in the
 395          * memory we allocated.
 396          */
 397         for (i = 0; i < dof->dofh_secnum; i++) {
 398                 if (dofs[i].dofs_type != DOF_SECT_URELHDR)
 399                         continue;
 400
 401                 /*LINTED*/
 402                 dofrh = (dof_relohdr_t *)((char *)dof + dofs[i].dofs_offset);
 403
 404                 s = &dofs[dofrh->dofr_strtab];
 405                 strtab = (char *)dof + s->dofs_offset;
 406                 bcopy(strtab + 1, dep->de_strtab + strtabsz, s->dofs_size);
 407                 base = strtabsz;
 408                 strtabsz += s->dofs_size - 1;
 409
 410                 s = &dofs[dofrh->dofr_relsec];
 411                 /*LINTED*/
 412                 dofr = (dof_relodesc_t *)((char *)dof + s->dofs_offset);
 413                 nrel = s->dofs_size / s->dofs_entsize;
 414
 415                 s = &dofs[dofrh->dofr_tgtsec];
 416
 417                 for (j = 0; j < nrel; j++) {
 418 #ifdef DOODAD
 419 #if defined(__arm__)
 420 /* XXX */
 421 #elif defined(__ia64__)
 422 /* XXX */
 423 #elif defined(__mips__)
 424 /* XXX */
 425 #elif defined(__powerpc__)
 426 /* XXX */
 427 #elif defined(__i386) || defined(__amd64)
 428                         rel->r_offset = s->dofs_offset +
 429                             dofr[j].dofr_offset;
 430                         rel->r_info = ELF64_R_INFO(count + dep->de_global,
 431                             R_AMD64_64);
 432 #elif defined(__sparc)
 433                         rel->r_offset = s->dofs_offset +
 434                             dofr[j].dofr_offset;
 435                         rel->r_info = ELF64_R_INFO(count + dep->de_global,
 436                             R_SPARC_64);
 437 #else
 438 #error unknown ISA
 439 #endif
 440 #endif
 441
 442                         sym->st_name = base + dofr[j].dofr_name - 1;
 443                         sym->st_value = 0;
 444                         sym->st_size = 0;
 445                         sym->st_info = GELF_ST_INFO(STB_GLOBAL, STT_FUNC);
 446                         sym->st_other = 0;
 447                         sym->st_shndx = SHN_UNDEF;
 448
 449                         rel++;
 450                         sym++;
 451                         count++;
 452                 }
 453         }
 454
 455         /*
 456          * Add a symbol for the DOF itself. We use a different symbol for
 457          * lazily and actively loaded DOF to make them easy to distinguish.
 458          */
 459         sym->st_name = strtabsz;
 460         sym->st_value = 0;
 461         sym->st_size = dof->dofh_filesz;
 462         sym->st_info = GELF_ST_INFO(STB_GLOBAL, STT_OBJECT);
 463         sym->st_other = 0;
 464         sym->st_shndx = ESHDR_DOF;
 465         sym++;
 466
 467         if (dtp->dt_lazyload) {
 468                 bcopy(DOFLAZYSTR, dep->de_strtab + strtabsz,
 469                     sizeof (DOFLAZYSTR));
 470                 strtabsz += sizeof (DOFLAZYSTR);
 471         } else {
 472                 bcopy(DOFSTR, dep->de_strtab + strtabsz, sizeof (DOFSTR));
 473                 strtabsz += sizeof (DOFSTR);
 474         }
 475
 476         assert(count == dep->de_nrel);
 477         assert(strtabsz == dep->de_strlen);
 478
 479         return (0);
 480 }
 481
 482 /*
 483  * Write out an ELF32 file prologue consisting of a header, section headers,
 484  * and a section header string table.  The DOF data will follow this prologue
 485  * and complete the contents of the given ELF file.
 486  */
 487 static int
 488 dump_elf32(dtrace_hdl_t *dtp, const dof_hdr_t *dof, int fd)
 489 {
 490         struct {
 491                 Elf32_Ehdr ehdr;
 492                 Elf32_Shdr shdr[ESHDR_NUM];
 493         } elf_file;
 494
 495         Elf32_Shdr *shp;
 496         Elf32_Off off;
 497         dof_elf32_t de;
 498         int ret = 0;
 499         uint_t nshdr;
 500
 501         if (prepare_elf32(dtp, dof, &de) != 0)
 502                 return (-1); /* errno is set for us */
 503
 504         /*
 505          * If there are no relocations, we only need enough sections for
 506          * the shstrtab and the DOF.
 507          */
 508         nshdr = de.de_nrel == 0 ? ESHDR_SYMTAB + 1 : ESHDR_NUM;
 509
 510         bzero(&elf_file, sizeof (elf_file));
 511
 512         elf_file.ehdr.e_ident[EI_MAG0] = ELFMAG0;
 513         elf_file.ehdr.e_ident[EI_MAG1] = ELFMAG1;
 514         elf_file.ehdr.e_ident[EI_MAG2] = ELFMAG2;
 515         elf_file.ehdr.e_ident[EI_MAG3] = ELFMAG3;
 516         elf_file.ehdr.e_ident[EI_VERSION] = EV_CURRENT;
 517         elf_file.ehdr.e_ident[EI_CLASS] = ELFCLASS32;
 518 #if BYTE_ORDER == _BIG_ENDIAN
 519         elf_file.ehdr.e_ident[EI_DATA] = ELFDATA2MSB;
 520 #else
 521         elf_file.ehdr.e_ident[EI_DATA] = ELFDATA2LSB;
 522 #endif
 523 #if defined(__FreeBSD__)
 524         elf_file.ehdr.e_ident[EI_OSABI] = ELFOSABI_FREEBSD;
 525 #endif
 526         elf_file.ehdr.e_type = ET_REL;
 527 #if defined(__arm__)
 528         elf_file.ehdr.e_machine = EM_ARM;
 529 #elif defined(__ia64__)
 530         elf_file.ehdr.e_machine = EM_IA_64;
 531 #elif defined(__mips__)
 532         elf_file.ehdr.e_machine = EM_MIPS;
 533 #elif defined(__powerpc__)
 534         elf_file.ehdr.e_machine = EM_PPC;
 535 #elif defined(__sparc)
 536         elf_file.ehdr.e_machine = EM_SPARC;
 537 #elif defined(__i386) || defined(__amd64)
 538         elf_file.ehdr.e_machine = EM_386;
 539 #endif
 540         elf_file.ehdr.e_version = EV_CURRENT;
 541         elf_file.ehdr.e_shoff = sizeof (Elf32_Ehdr);
 542         elf_file.ehdr.e_ehsize = sizeof (Elf32_Ehdr);
 543         elf_file.ehdr.e_phentsize = sizeof (Elf32_Phdr);
 544         elf_file.ehdr.e_shentsize = sizeof (Elf32_Shdr);
 545         elf_file.ehdr.e_shnum = nshdr;
 546         elf_file.ehdr.e_shstrndx = ESHDR_SHSTRTAB;
 547         off = sizeof (elf_file) + nshdr * sizeof (Elf32_Shdr);
 548
 549         shp = &elf_file.shdr[ESHDR_SHSTRTAB];
 550         shp->sh_name = 1; /* DTRACE_SHSTRTAB32[1] = ".shstrtab" */
 551         shp->sh_type = SHT_STRTAB;
 552         shp->sh_offset = off;
 553         shp->sh_size = sizeof (DTRACE_SHSTRTAB32);
 554         shp->sh_addralign = sizeof (char);
 555         off = P2ROUNDUP(shp->sh_offset + shp->sh_size, 8);
 556
 557         shp = &elf_file.shdr[ESHDR_DOF];
 558         shp->sh_name = 11; /* DTRACE_SHSTRTAB32[11] = ".SUNW_dof" */
 559         shp->sh_flags = SHF_ALLOC;
 560         shp->sh_type = SHT_SUNW_dof;
 561         shp->sh_offset = off;
 562         shp->sh_size = dof->dofh_filesz;
 563         shp->sh_addralign = 8;
 564         off = shp->sh_offset + shp->sh_size;
 565
 566         shp = &elf_file.shdr[ESHDR_STRTAB];
 567         shp->sh_name = 21; /* DTRACE_SHSTRTAB32[21] = ".strtab" */
 568         shp->sh_flags = SHF_ALLOC;
 569         shp->sh_type = SHT_STRTAB;
 570         shp->sh_offset = off;
 571         shp->sh_size = de.de_strlen;
 572         shp->sh_addralign = sizeof (char);
 573         off = P2ROUNDUP(shp->sh_offset + shp->sh_size, 4);
 574
 575         shp = &elf_file.shdr[ESHDR_SYMTAB];
 576         shp->sh_name = 29; /* DTRACE_SHSTRTAB32[29] = ".symtab" */
 577         shp->sh_flags = SHF_ALLOC;
 578         shp->sh_type = SHT_SYMTAB;
 579         shp->sh_entsize = sizeof (Elf32_Sym);
 580         shp->sh_link = ESHDR_STRTAB;
 581         shp->sh_offset = off;
 582         shp->sh_info = de.de_global;
 583         shp->sh_size = de.de_nsym * sizeof (Elf32_Sym);
 584         shp->sh_addralign = 4;
 585         off = P2ROUNDUP(shp->sh_offset + shp->sh_size, 4);
 586
 587         if (de.de_nrel == 0) {
 588                 if (dt_write(dtp, fd, &elf_file,
 589                     sizeof (elf_file)) != sizeof (elf_file) ||
 590                     PWRITE_SCN(ESHDR_SHSTRTAB, DTRACE_SHSTRTAB32) ||
 591                     PWRITE_SCN(ESHDR_STRTAB, de.de_strtab) ||
 592                     PWRITE_SCN(ESHDR_SYMTAB, de.de_sym) ||
 593                     PWRITE_SCN(ESHDR_DOF, dof)) {
 594                         ret = dt_set_errno(dtp, errno);
 595                 }
 596         } else {
 597                 shp = &elf_file.shdr[ESHDR_REL];
 598                 shp->sh_name = 37; /* DTRACE_SHSTRTAB32[37] = ".rel.SUNW_dof" */
 599                 shp->sh_flags = SHF_ALLOC;
 600 #ifdef __sparc
 601                 shp->sh_type = SHT_RELA;
 602 #else
 603                 shp->sh_type = SHT_REL;
 604 #endif
 605                 shp->sh_entsize = sizeof (de.de_rel[0]);
 606                 shp->sh_link = ESHDR_SYMTAB;
 607                 shp->sh_info = ESHDR_DOF;
 608                 shp->sh_offset = off;
 609                 shp->sh_size = de.de_nrel * sizeof (de.de_rel[0]);
 610                 shp->sh_addralign = 4;
 611
 612                 if (dt_write(dtp, fd, &elf_file,
 613                     sizeof (elf_file)) != sizeof (elf_file) ||
 614                     PWRITE_SCN(ESHDR_SHSTRTAB, DTRACE_SHSTRTAB32) ||
 615                     PWRITE_SCN(ESHDR_STRTAB, de.de_strtab) ||
 616                     PWRITE_SCN(ESHDR_SYMTAB, de.de_sym) ||
 617                     PWRITE_SCN(ESHDR_REL, de.de_rel) ||
 618                     PWRITE_SCN(ESHDR_DOF, dof)) {
 619                         ret = dt_set_errno(dtp, errno);
 620                 }
 621         }
 622
 623         free(de.de_strtab);
 624         free(de.de_sym);
 625         free(de.de_rel);
 626
 627         return (ret);
 628 }
 629
 630 /*
 631  * Write out an ELF64 file prologue consisting of a header, section headers,
 632  * and a section header string table.  The DOF data will follow this prologue
 633  * and complete the contents of the given ELF file.
 634  */
 635 static int
 636 dump_elf64(dtrace_hdl_t *dtp, const dof_hdr_t *dof, int fd)
 637 {
 638         struct {
 639                 Elf64_Ehdr ehdr;
 640                 Elf64_Shdr shdr[ESHDR_NUM];
 641         } elf_file;
 642
 643         Elf64_Shdr *shp;
 644         Elf64_Off off;
 645         dof_elf64_t de;
 646         int ret = 0;
 647         uint_t nshdr;
 648
 649         if (prepare_elf64(dtp, dof, &de) != 0)
 650                 return (-1); /* errno is set for us */
 651
 652         /*
 653          * If there are no relocations, we only need enough sections for
 654          * the shstrtab and the DOF.
 655          */
 656         nshdr = de.de_nrel == 0 ? ESHDR_SYMTAB + 1 : ESHDR_NUM;
 657
 658         bzero(&elf_file, sizeof (elf_file));
 659
 660         elf_file.ehdr.e_ident[EI_MAG0] = ELFMAG0;
 661         elf_file.ehdr.e_ident[EI_MAG1] = ELFMAG1;
 662         elf_file.ehdr.e_ident[EI_MAG2] = ELFMAG2;
 663         elf_file.ehdr.e_ident[EI_MAG3] = ELFMAG3;
 664         elf_file.ehdr.e_ident[EI_VERSION] = EV_CURRENT;
 665         elf_file.ehdr.e_ident[EI_CLASS] = ELFCLASS64;
 666 #if BYTE_ORDER == _BIG_ENDIAN
 667         elf_file.ehdr.e_ident[EI_DATA] = ELFDATA2MSB;
 668 #else
 669         elf_file.ehdr.e_ident[EI_DATA] = ELFDATA2LSB;
 670 #endif
 671 #if defined(__FreeBSD__)
 672         elf_file.ehdr.e_ident[EI_OSABI] = ELFOSABI_FREEBSD;
 673 #endif
 674         elf_file.ehdr.e_type = ET_REL;
 675 #if defined(__arm__)
 676         elf_file.ehdr.e_machine = EM_ARM;
 677 #elif defined(__ia64__)
 678         elf_file.ehdr.e_machine = EM_IA_64;
 679 #elif defined(__mips__)
 680         elf_file.ehdr.e_machine = EM_MIPS;
 681 #elif defined(__powerpc__)
 682         elf_file.ehdr.e_machine = EM_PPC;
 683 #elif defined(__sparc)
 684         elf_file.ehdr.e_machine = EM_SPARCV9;
 685 #elif defined(__i386) || defined(__amd64)
 686         elf_file.ehdr.e_machine = EM_AMD64;
 687 #endif
 688         elf_file.ehdr.e_version = EV_CURRENT;
 689         elf_file.ehdr.e_shoff = sizeof (Elf64_Ehdr);
 690         elf_file.ehdr.e_ehsize = sizeof (Elf64_Ehdr);
 691         elf_file.ehdr.e_phentsize = sizeof (Elf64_Phdr);
 692         elf_file.ehdr.e_shentsize = sizeof (Elf64_Shdr);
 693         elf_file.ehdr.e_shnum = nshdr;
 694         elf_file.ehdr.e_shstrndx = ESHDR_SHSTRTAB;
 695         off = sizeof (elf_file) + nshdr * sizeof (Elf64_Shdr);
 696
 697         shp = &elf_file.shdr[ESHDR_SHSTRTAB];
 698         shp->sh_name = 1; /* DTRACE_SHSTRTAB64[1] = ".shstrtab" */
 699         shp->sh_type = SHT_STRTAB;
 700         shp->sh_offset = off;
 701         shp->sh_size = sizeof (DTRACE_SHSTRTAB64);
 702         shp->sh_addralign = sizeof (char);
 703         off = P2ROUNDUP(shp->sh_offset + shp->sh_size, 8);
 704
 705         shp = &elf_file.shdr[ESHDR_DOF];
 706         shp->sh_name = 11; /* DTRACE_SHSTRTAB64[11] = ".SUNW_dof" */
 707         shp->sh_flags = SHF_ALLOC;
 708         shp->sh_type = SHT_SUNW_dof;
 709         shp->sh_offset = off;
 710         shp->sh_size = dof->dofh_filesz;
 711         shp->sh_addralign = 8;
 712         off = shp->sh_offset + shp->sh_size;
 713
 714         shp = &elf_file.shdr[ESHDR_STRTAB];
 715         shp->sh_name = 21; /* DTRACE_SHSTRTAB64[21] = ".strtab" */
 716         shp->sh_flags = SHF_ALLOC;
 717         shp->sh_type = SHT_STRTAB;
 718         shp->sh_offset = off;
 719         shp->sh_size = de.de_strlen;
 720         shp->sh_addralign = sizeof (char);
 721         off = P2ROUNDUP(shp->sh_offset + shp->sh_size, 8);
 722
 723         shp = &elf_file.shdr[ESHDR_SYMTAB];
 724         shp->sh_name = 29; /* DTRACE_SHSTRTAB64[29] = ".symtab" */
 725         shp->sh_flags = SHF_ALLOC;
 726         shp->sh_type = SHT_SYMTAB;
 727         shp->sh_entsize = sizeof (Elf64_Sym);
 728         shp->sh_link = ESHDR_STRTAB;
 729         shp->sh_offset = off;
 730         shp->sh_info = de.de_global;
 731         shp->sh_size = de.de_nsym * sizeof (Elf64_Sym);
 732         shp->sh_addralign = 8;
 733         off = P2ROUNDUP(shp->sh_offset + shp->sh_size, 8);
 734
 735         if (de.de_nrel == 0) {
 736                 if (dt_write(dtp, fd, &elf_file,
 737                     sizeof (elf_file)) != sizeof (elf_file) ||
 738                     PWRITE_SCN(ESHDR_SHSTRTAB, DTRACE_SHSTRTAB64) ||
 739                     PWRITE_SCN(ESHDR_STRTAB, de.de_strtab) ||
 740                     PWRITE_SCN(ESHDR_SYMTAB, de.de_sym) ||
 741                     PWRITE_SCN(ESHDR_DOF, dof)) {
 742                         ret = dt_set_errno(dtp, errno);
 743                 }
 744         } else {
 745                 shp = &elf_file.shdr[ESHDR_REL];
 746                 shp->sh_name = 37; /* DTRACE_SHSTRTAB64[37] = ".rel.SUNW_dof" */
 747                 shp->sh_flags = SHF_ALLOC;
 748                 shp->sh_type = SHT_RELA;
 749                 shp->sh_entsize = sizeof (de.de_rel[0]);
 750                 shp->sh_link = ESHDR_SYMTAB;
 751                 shp->sh_info = ESHDR_DOF;
 752                 shp->sh_offset = off;
 753                 shp->sh_size = de.de_nrel * sizeof (de.de_rel[0]);
 754                 shp->sh_addralign = 8;
 755
 756                 if (dt_write(dtp, fd, &elf_file,
 757                     sizeof (elf_file)) != sizeof (elf_file) ||
 758                     PWRITE_SCN(ESHDR_SHSTRTAB, DTRACE_SHSTRTAB64) ||
 759                     PWRITE_SCN(ESHDR_STRTAB, de.de_strtab) ||
 760                     PWRITE_SCN(ESHDR_SYMTAB, de.de_sym) ||
 761                     PWRITE_SCN(ESHDR_REL, de.de_rel) ||
 762                     PWRITE_SCN(ESHDR_DOF, dof)) {
 763                         ret = dt_set_errno(dtp, errno);
 764                 }
 765         }
 766
 767         free(de.de_strtab);
 768         free(de.de_sym);
 769         free(de.de_rel);
 770
 771         return (ret);
 772 }
 773
 774 static int
 775 dt_symtab_lookup(Elf_Data *data_sym, int nsym, uintptr_t addr, uint_t shn,
 776     GElf_Sym *sym)
 777 {
 778         int i, ret = -1;
 779         GElf_Sym s;
 780
 781         for (i = 0; i < nsym && gelf_getsym(data_sym, i, sym) != NULL; i++) {
 782                 if (GELF_ST_TYPE(sym->st_info) == STT_FUNC &&
 783                     shn == sym->st_shndx &&
 784                     sym->st_value <= addr &&
 785                     addr < sym->st_value + sym->st_size) {
 786                         if (GELF_ST_BIND(sym->st_info) == STB_GLOBAL)
 787                                 return (0);
 788
 789                         ret = 0;
 790                         s = *sym;
 791                 }
 792         }
 793
 794         if (ret == 0)
 795                 *sym = s;
 796         return (ret);
 797 }
 798
 799 #if defined(__arm__)
 800 /* XXX */
 801 static int
 802 dt_modtext(dtrace_hdl_t *dtp, char *p, int isenabled, GElf_Rela *rela,
 803     uint32_t *off)
 804 {
 805 printf("%s:%s(%d): DOODAD\n",__FUNCTION__,__FILE__,__LINE__);
 806         return (0);
 807 }
 808 #elif defined(__ia64__)
 809 /* XXX */
 810 static int
 811 dt_modtext(dtrace_hdl_t *dtp, char *p, int isenabled, GElf_Rela *rela,
 812     uint32_t *off)
 813 {
 814 printf("%s:%s(%d): DOODAD\n",__FUNCTION__,__FILE__,__LINE__);
 815         return (0);
 816 }
 817 #elif defined(__mips__)
 818 /* XXX */
 819 static int
 820 dt_modtext(dtrace_hdl_t *dtp, char *p, int isenabled, GElf_Rela *rela,
 821     uint32_t *off)
 822 {
 823 printf("%s:%s(%d): DOODAD\n",__FUNCTION__,__FILE__,__LINE__);
 824         return (0);
 825 }
 826 #elif defined(__powerpc__)
 827 /* XXX */
 828 static int
 829 dt_modtext(dtrace_hdl_t *dtp, char *p, int isenabled, GElf_Rela *rela,
 830     uint32_t *off)
 831 {
 832 printf("%s:%s(%d): DOODAD\n",__FUNCTION__,__FILE__,__LINE__);
 833         return (0);
 834 }
 835
 836 #elif defined(__sparc)
 837
 838 #define DT_OP_RET               0x81c7e008
 839 #define DT_OP_NOP               0x01000000
 840 #define DT_OP_CALL              0x40000000
 841 #define DT_OP_CLR_O0            0x90102000
 842
 843 #define DT_IS_MOV_O7(inst)      (((inst) & 0xffffe000) == 0x9e100000)
 844 #define DT_IS_RESTORE(inst)     (((inst) & 0xc1f80000) == 0x81e80000)
 845 #define DT_IS_RETL(inst)        (((inst) & 0xfff83fff) == 0x81c02008)
 846
 847 #define DT_RS2(inst)            ((inst) & 0x1f)
 848 #define DT_MAKE_RETL(reg)       (0x81c02008 | ((reg) << 14))
 849
 850 /*ARGSUSED*/
 851 static int
 852 dt_modtext(dtrace_hdl_t *dtp, char *p, int isenabled, GElf_Rela *rela,
 853     uint32_t *off)
 854 {
 855         uint32_t *ip;
 856
 857         if ((rela->r_offset & (sizeof (uint32_t) - 1)) != 0)
 858                 return (-1);
 859
 860         /*LINTED*/
 861         ip = (uint32_t *)(p + rela->r_offset);
 862
 863         /*
 864          * We only know about some specific relocation types.
 865          */
 866         if (GELF_R_TYPE(rela->r_info) != R_SPARC_WDISP30 &&
 867             GELF_R_TYPE(rela->r_info) != R_SPARC_WPLT30)
 868                 return (-1);
 869
 870         /*
 871          * We may have already processed this object file in an earlier linker
 872          * invocation. Check to see if the present instruction sequence matches
 873          * the one we would install below.
 874          */
 875         if (isenabled) {
 876                 if (ip[0] == DT_OP_NOP) {
 877                         (*off) += sizeof (ip[0]);
 878                         return (0);
 879                 }
 880         } else {
 881                 if (DT_IS_RESTORE(ip[1])) {
 882                         if (ip[0] == DT_OP_RET) {
 883                                 (*off) += sizeof (ip[0]);
 884                                 return (0);
 885                         }
 886                 } else if (DT_IS_MOV_O7(ip[1])) {
 887                         if (DT_IS_RETL(ip[0]))
 888                                 return (0);
 889                 } else {
 890                         if (ip[0] == DT_OP_NOP) {
 891                                 (*off) += sizeof (ip[0]);
 892                                 return (0);
 893                         }
 894                 }
 895         }
 896
 897         /*
 898          * We only expect call instructions with a displacement of 0.
 899          */
 900         if (ip[0] != DT_OP_CALL) {
 901                 dt_dprintf("found %x instead of a call instruction at %llx\n",
 902                     ip[0], (u_longlong_t)rela->r_offset);
 903                 return (-1);
 904         }
 905
 906         if (isenabled) {
 907                 /*
 908                  * It would necessarily indicate incorrect usage if an is-
 909                  * enabled probe were tail-called so flag that as an error.
 910                  * It's also potentially (very) tricky to handle gracefully,
 911                  * but could be done if this were a desired use scenario.
 912                  */
 913                 if (DT_IS_RESTORE(ip[1]) || DT_IS_MOV_O7(ip[1])) {
 914                         dt_dprintf("tail call to is-enabled probe at %llx\n",
 915                             (u_longlong_t)rela->r_offset);
 916                         return (-1);
 917                 }
 918
 919
 920                 /*
 921                  * On SPARC, we take advantage of the fact that the first
 922                  * argument shares the same register as for the return value.
 923                  * The macro handles the work of zeroing that register so we
 924                  * don't need to do anything special here. We instrument the
 925                  * instruction in the delay slot as we'll need to modify the
 926                  * return register after that instruction has been emulated.
 927                  */
 928                 ip[0] = DT_OP_NOP;
 929                 (*off) += sizeof (ip[0]);
 930         } else {
 931                 /*
 932                  * If the call is followed by a restore, it's a tail call so
 933                  * change the call to a ret. If the call if followed by a mov
 934                  * of a register into %o7, it's a tail call in leaf context
 935                  * so change the call to a retl-like instruction that returns
 936                  * to that register value + 8 (rather than the typical %o7 +
 937                  * 8); the delay slot instruction is left, but should have no
 938                  * effect. Otherwise we change the call to be a nop. We
 939                  * identify the subsequent instruction as the probe point in
 940                  * all but the leaf tail-call case to ensure that arguments to
 941                  * the probe are complete and consistent. An astute, though
 942                  * largely hypothetical, observer would note that there is the
 943                  * possibility of a false-positive probe firing if the function
 944                  * contained a branch to the instruction in the delay slot of
 945                  * the call. Fixing this would require significant in-kernel
 946                  * modifications, and isn't worth doing until we see it in the
 947                  * wild.
 948                  */
 949                 if (DT_IS_RESTORE(ip[1])) {
 950                         ip[0] = DT_OP_RET;
 951                         (*off) += sizeof (ip[0]);
 952                 } else if (DT_IS_MOV_O7(ip[1])) {
 953                         ip[0] = DT_MAKE_RETL(DT_RS2(ip[1]));
 954                 } else {
 955                         ip[0] = DT_OP_NOP;
 956                         (*off) += sizeof (ip[0]);
 957                 }
 958         }
 959
 960         return (0);
 961 }
 962
 963 #elif defined(__i386) || defined(__amd64)
 964
 965 #define DT_OP_NOP               0x90
 966 #define DT_OP_RET               0xc3
 967 #define DT_OP_CALL              0xe8
 968 #define DT_OP_JMP32             0xe9
 969 #define DT_OP_REX_RAX           0x48
 970 #define DT_OP_XOR_EAX_0         0x33
 971 #define DT_OP_XOR_EAX_1         0xc0
 972
 973 static int
 974 dt_modtext(dtrace_hdl_t *dtp, char *p, int isenabled, GElf_Rela *rela,
 975     uint32_t *off)
 976 {
 977         uint8_t *ip = (uint8_t *)(p + rela->r_offset - 1);
 978         uint8_t ret;
 979
 980         /*
 981          * On x86, the first byte of the instruction is the call opcode and
 982          * the next four bytes are the 32-bit address; the relocation is for
 983          * the address operand. We back up the offset to the first byte of
 984          * the instruction. For is-enabled probes, we later advance the offset
 985          * so that it hits the first nop in the instruction sequence.
 986          */
 987         (*off) -= 1;
 988
 989         /*
 990          * We only know about some specific relocation types. Luckily
 991          * these types have the same values on both 32-bit and 64-bit
 992          * x86 architectures.
 993          */
 994         if (GELF_R_TYPE(rela->r_info) != R_386_PC32 &&
 995             GELF_R_TYPE(rela->r_info) != R_386_PLT32)
 996                 return (-1);
 997
 998         /*
 999          * We may have already processed this object file in an earlier linker
1000          * invocation. Check to see if the present instruction sequence matches
1001          * the one we would install. For is-enabled probes, we advance the
1002          * offset to the first nop instruction in the sequence to match the
1003          * text modification code below.
1004          */
1005         if (!isenabled) {
1006                 if ((ip[0] == DT_OP_NOP || ip[0] == DT_OP_RET) &&
1007                     ip[1] == DT_OP_NOP && ip[2] == DT_OP_NOP &&
1008                     ip[3] == DT_OP_NOP && ip[4] == DT_OP_NOP)
1009                         return (0);
1010         } else if (dtp->dt_oflags & DTRACE_O_LP64) {
1011                 if (ip[0] == DT_OP_REX_RAX &&
1012                     ip[1] == DT_OP_XOR_EAX_0 && ip[2] == DT_OP_XOR_EAX_1 &&
1013                     (ip[3] == DT_OP_NOP || ip[3] == DT_OP_RET) &&
1014                     ip[4] == DT_OP_NOP) {
1015                         (*off) += 3;
1016                         return (0);
1017                 }
1018         } else {
1019                 if (ip[0] == DT_OP_XOR_EAX_0 && ip[1] == DT_OP_XOR_EAX_1 &&
1020                     (ip[2] == DT_OP_NOP || ip[2] == DT_OP_RET) &&
1021                     ip[3] == DT_OP_NOP && ip[4] == DT_OP_NOP) {
1022                         (*off) += 2;
1023                         return (0);
1024                 }
1025         }
1026
1027         /*
1028          * We expect either a call instrution with a 32-bit displacement or a
1029          * jmp instruction with a 32-bit displacement acting as a tail-call.
1030          */
1031         if (ip[0] != DT_OP_CALL && ip[0] != DT_OP_JMP32) {
1032                 dt_dprintf("found %x instead of a call or jmp instruction at "
1033                     "%llx\n", ip[0], (u_longlong_t)rela->r_offset);
1034                 return (-1);
1035         }
1036
1037         ret = (ip[0] == DT_OP_JMP32) ? DT_OP_RET : DT_OP_NOP;
1038
1039         /*
1040          * Establish the instruction sequence -- all nops for probes, and an
1041          * instruction to clear the return value register (%eax/%rax) followed
1042          * by nops for is-enabled probes. For is-enabled probes, we advance
1043          * the offset to the first nop. This isn't stricly necessary but makes
1044          * for more readable disassembly when the probe is enabled.
1045          */
1046         if (!isenabled) {
1047                 ip[0] = ret;
1048                 ip[1] = DT_OP_NOP;
1049                 ip[2] = DT_OP_NOP;
1050                 ip[3] = DT_OP_NOP;
1051                 ip[4] = DT_OP_NOP;
1052         } else if (dtp->dt_oflags & DTRACE_O_LP64) {
1053                 ip[0] = DT_OP_REX_RAX;
1054                 ip[1] = DT_OP_XOR_EAX_0;
1055                 ip[2] = DT_OP_XOR_EAX_1;
1056                 ip[3] = ret;
1057                 ip[4] = DT_OP_NOP;
1058                 (*off) += 3;
1059         } else {
1060                 ip[0] = DT_OP_XOR_EAX_0;
1061                 ip[1] = DT_OP_XOR_EAX_1;
1062                 ip[2] = ret;
1063                 ip[3] = DT_OP_NOP;
1064                 ip[4] = DT_OP_NOP;
1065                 (*off) += 2;
1066         }
1067
1068         return (0);
1069 }
1070
1071 #else
1072 #error unknown ISA
1073 #endif
1074
1075 /*PRINTFLIKE5*/
1076 static int
1077 dt_link_error(dtrace_hdl_t *dtp, Elf *elf, int fd, dt_link_pair_t *bufs,
1078     const char *format, ...)
1079 {
1080         va_list ap;
1081         dt_link_pair_t *pair;
1082
1083         va_start(ap, format);
1084         dt_set_errmsg(dtp, NULL, NULL, NULL, 0, format, ap);
1085         va_end(ap);
1086
1087         if (elf != NULL)
1088                 (void) elf_end(elf);
1089
1090         if (fd >= 0)
1091                 (void) close(fd);
1092
1093         while ((pair = bufs) != NULL) {
1094                 bufs = pair->dlp_next;
1095                 dt_free(dtp, pair->dlp_str);
1096                 dt_free(dtp, pair->dlp_sym);
1097                 dt_free(dtp, pair);
1098         }
1099
1100         return (dt_set_errno(dtp, EDT_COMPILER));
1101 }
1102
1103 static int
1104 process_obj(dtrace_hdl_t *dtp, const char *obj, int *eprobesp)
1105 {
1106         static const char dt_prefix[] = "__dtrace";
1107         static const char dt_enabled[] = "enabled";
1108         static const char dt_symprefix[] = "$dtrace";
1109         static const char dt_symfmt[] = "%s%ld.%s";
1110         int fd, i, ndx, eprobe, mod = 0;
1111         Elf *elf = NULL;
1112         GElf_Ehdr ehdr;
1113         Elf_Scn *scn_rel, *scn_sym, *scn_str, *scn_tgt;
1114         Elf_Data *data_rel, *data_sym, *data_str, *data_tgt;
1115         GElf_Shdr shdr_rel, shdr_sym, shdr_str, shdr_tgt;
1116         GElf_Sym rsym, fsym, dsym;
1117         GElf_Rela rela;
1118         char *s, *p, *r;
1119         char pname[DTRACE_PROVNAMELEN];
1120         dt_provider_t *pvp;
1121         dt_probe_t *prp;
1122         uint32_t off, eclass, emachine1, emachine2;
1123         size_t symsize, nsym, isym, istr, len;
1124         key_t objkey;
1125         dt_link_pair_t *pair, *bufs = NULL;
1126         dt_strtab_t *strtab;
1127
1128         if ((fd = open64(obj, O_RDWR)) == -1) {
1129                 return (dt_link_error(dtp, elf, fd, bufs,
1130                     "failed to open %s: %s", obj, strerror(errno)));
1131         }
1132
1133         if ((elf = elf_begin(fd, ELF_C_RDWR, NULL)) == NULL) {
1134                 return (dt_link_error(dtp, elf, fd, bufs,
1135                     "failed to process %s: %s", obj, elf_errmsg(elf_errno())));
1136         }
1137
1138         switch (elf_kind(elf)) {
1139         case ELF_K_ELF:
1140                 break;
1141         case ELF_K_AR:
1142                 return (dt_link_error(dtp, elf, fd, bufs, "archives are not "
1143                     "permitted; use the contents of the archive instead: %s",
1144                     obj));
1145         default:
1146                 return (dt_link_error(dtp, elf, fd, bufs,
1147                     "invalid file type: %s", obj));
1148         }
1149
1150         if (gelf_getehdr(elf, &ehdr) == NULL) {
1151                 return (dt_link_error(dtp, elf, fd, bufs, "corrupt file: %s",
1152                     obj));
1153         }
1154
1155         if (dtp->dt_oflags & DTRACE_O_LP64) {
1156                 eclass = ELFCLASS64;
1157 #if defined(__ia64__)
1158                 emachine1 = emachine2 = EM_IA_64;
1159 #elif defined(__mips__)
1160                 emachine1 = emachine2 = EM_MIPS;
1161 #elif defined(__powerpc__)
1162                 emachine1 = emachine2 = EM_PPC64;
1163 #elif defined(__sparc)
1164                 emachine1 = emachine2 = EM_SPARCV9;
1165 #elif defined(__i386) || defined(__amd64)
1166                 emachine1 = emachine2 = EM_AMD64;
1167 #endif
1168                 symsize = sizeof (Elf64_Sym);
1169         } else {
1170                 eclass = ELFCLASS32;
1171 #if defined(__arm__)
1172                 emachine1 = emachine2 = EM_ARM;
1173 #elif defined(__mips__)
1174                 emachine1 = emachine2 = EM_MIPS;
1175 #elif defined(__powerpc__)
1176                 emachine1 = emachine2 = EM_PPC;
1177 #elif defined(__sparc)
1178                 emachine1 = EM_SPARC;
1179                 emachine2 = EM_SPARC32PLUS;
1180 #elif defined(__i386) || defined(__amd64) || defined(__ia64__)
1181                 emachine1 = emachine2 = EM_386;
1182 #endif
1183                 symsize = sizeof (Elf32_Sym);
1184         }
1185
1186         if (ehdr.e_ident[EI_CLASS] != eclass) {
1187                 return (dt_link_error(dtp, elf, fd, bufs,
1188                     "incorrect ELF class for object file: %s", obj));
1189         }
1190
1191         if (ehdr.e_machine != emachine1 && ehdr.e_machine != emachine2) {
1192                 return (dt_link_error(dtp, elf, fd, bufs,
1193                     "incorrect ELF machine type for object file: %s", obj));
1194         }
1195
1196         /*
1197          * We use this token as a relatively unique handle for this file on the
1198          * system in order to disambiguate potential conflicts between files of
1199          * the same name which contain identially named local symbols.
1200          */
1201         if ((objkey = ftok(obj, 0)) == (key_t)-1) {
1202                 return (dt_link_error(dtp, elf, fd, bufs,
1203                     "failed to generate unique key for object file: %s", obj));
1204         }
1205
1206         scn_rel = NULL;
1207         while ((scn_rel = elf_nextscn(elf, scn_rel)) != NULL) {
1208                 if (gelf_getshdr(scn_rel, &shdr_rel) == NULL)
1209                         goto err;
1210
1211                 /*
1212                  * Skip any non-relocation sections.
1213                  */
1214                 if (shdr_rel.sh_type != SHT_RELA && shdr_rel.sh_type != SHT_REL)
1215                         continue;
1216
1217                 if ((data_rel = elf_getdata(scn_rel, NULL)) == NULL)
1218                         goto err;
1219
1220                 /*
1221                  * Grab the section, section header and section data for the
1222                  * symbol table that this relocation section references.
1223                  */
1224                 if ((scn_sym = elf_getscn(elf, shdr_rel.sh_link)) == NULL ||
1225                     gelf_getshdr(scn_sym, &shdr_sym) == NULL ||
1226                     (data_sym = elf_getdata(scn_sym, NULL)) == NULL)
1227                         goto err;
1228
1229                 /*
1230                  * Ditto for that symbol table's string table.
1231                  */
1232                 if ((scn_str = elf_getscn(elf, shdr_sym.sh_link)) == NULL ||
1233                     gelf_getshdr(scn_str, &shdr_str) == NULL ||
1234                     (data_str = elf_getdata(scn_str, NULL)) == NULL)
1235                         goto err;
1236
1237                 /*
1238                  * Grab the section, section header and section data for the
1239                  * target section for the relocations. For the relocations
1240                  * we're looking for -- this will typically be the text of the
1241                  * object file.
1242                  */
1243                 if ((scn_tgt = elf_getscn(elf, shdr_rel.sh_info)) == NULL ||
1244                     gelf_getshdr(scn_tgt, &shdr_tgt) == NULL ||
1245                     (data_tgt = elf_getdata(scn_tgt, NULL)) == NULL)
1246                         goto err;
1247
1248                 /*
1249                  * We're looking for relocations to symbols matching this form:
1250                  *
1251                  *   __dtrace[enabled]_<prov>___<probe>
1252                  *
1253                  * For the generated object, we need to record the location
1254                  * identified by the relocation, and create a new relocation
1255                  * in the generated object that will be resolved at link time
1256                  * to the location of the function in which the probe is
1257                  * embedded. In the target object, we change the matched symbol
1258                  * so that it will be ignored at link time, and we modify the
1259                  * target (text) section to replace the call instruction with
1260                  * one or more nops.
1261                  *
1262                  * If the function containing the probe is locally scoped
1263                  * (static), we create an alias used by the relocation in the
1264                  * generated object. The alias, a new symbol, will be global
1265                  * (so that the relocation from the generated object can be
1266                  * resolved), and hidden (so that it is converted to a local
1267                  * symbol at link time). Such aliases have this form:
1268                  *
1269                  *   $dtrace<key>.<function>
1270                  *
1271                  * We take a first pass through all the relocations to
1272                  * populate our string table and count the number of extra
1273                  * symbols we'll require.
1274                  */
1275                 strtab = dt_strtab_create(1);
1276                 nsym = 0;
1277                 isym = data_sym->d_size / symsize;
1278                 istr = data_str->d_size;
1279
1280                 for (i = 0; i < shdr_rel.sh_size / shdr_rel.sh_entsize; i++) {
1281
1282                         if (shdr_rel.sh_type == SHT_RELA) {
1283                                 if (gelf_getrela(data_rel, i, &rela) == NULL)
1284                                         continue;
1285                         } else {
1286                                 GElf_Rel rel;
1287                                 if (gelf_getrel(data_rel, i, &rel) == NULL)
1288                                         continue;
1289                                 rela.r_offset = rel.r_offset;
1290                                 rela.r_info = rel.r_info;
1291                                 rela.r_addend = 0;
1292                         }
1293
1294                         if (gelf_getsym(data_sym, GELF_R_SYM(rela.r_info),
1295                             &rsym) == NULL) {
1296                                 dt_strtab_destroy(strtab);
1297                                 goto err;
1298                         }
1299
1300                         s = (char *)data_str->d_buf + rsym.st_name;
1301
1302                         if (strncmp(s, dt_prefix, sizeof (dt_prefix) - 1) != 0)
1303                                 continue;
1304
1305                         if (dt_symtab_lookup(data_sym, isym, rela.r_offset,
1306                             shdr_rel.sh_info, &fsym) != 0) {
1307                                 dt_strtab_destroy(strtab);
1308                                 goto err;
1309                         }
1310
1311                         if (GELF_ST_BIND(fsym.st_info) != STB_LOCAL)
1312                                 continue;
1313
1314                         if (fsym.st_name > data_str->d_size) {
1315                                 dt_strtab_destroy(strtab);
1316                                 goto err;
1317                         }
1318
1319                         s = (char *)data_str->d_buf + fsym.st_name;
1320
1321                         /*
1322                          * If this symbol isn't of type function, we've really
1323                          * driven off the rails or the object file is corrupt.
1324                          */
1325                         if (GELF_ST_TYPE(fsym.st_info) != STT_FUNC) {
1326                                 dt_strtab_destroy(strtab);
1327                                 return (dt_link_error(dtp, elf, fd, bufs,
1328                                     "expected %s to be of type function", s));
1329                         }
1330
1331                         len = snprintf(NULL, 0, dt_symfmt, dt_symprefix,
1332                             objkey, s) + 1;
1333                         if ((p = dt_alloc(dtp, len)) == NULL) {
1334                                 dt_strtab_destroy(strtab);
1335                                 goto err;
1336                         }
1337                         (void) snprintf(p, len, dt_symfmt, dt_symprefix,
1338                             objkey, s);
1339
1340                         if (dt_strtab_index(strtab, p) == -1) {
1341                                 nsym++;
1342                                 (void) dt_strtab_insert(strtab, p);
1343                         }
1344
1345                         dt_free(dtp, p);
1346                 }
1347
1348                 /*
1349                  * If needed, allocate the additional space for the symbol
1350                  * table and string table copying the old data into the new
1351                  * buffers, and marking the buffers as dirty. We inject those
1352                  * newly allocated buffers into the libelf data structures, but
1353                  * are still responsible for freeing them once we're done with
1354                  * the elf handle.
1355                  */
1356                 if (nsym > 0) {
1357                         /*
1358                          * The first byte of the string table is reserved for
1359                          * the \0 entry.
1360                          */
1361                         len = dt_strtab_size(strtab) - 1;
1362
1363                         assert(len > 0);
1364                         assert(dt_strtab_index(strtab, "") == 0);
1365
1366                         dt_strtab_destroy(strtab);
1367
1368                         if ((pair = dt_alloc(dtp, sizeof (*pair))) == NULL)
1369                                 goto err;
1370
1371                         if ((pair->dlp_str = dt_alloc(dtp, data_str->d_size +
1372                             len)) == NULL) {
1373                                 dt_free(dtp, pair);
1374                                 goto err;
1375                         }
1376
1377                         if ((pair->dlp_sym = dt_alloc(dtp, data_sym->d_size +
1378                             nsym * symsize)) == NULL) {
1379                                 dt_free(dtp, pair->dlp_str);
1380                                 dt_free(dtp, pair);
1381                                 goto err;
1382                         }
1383
1384                         pair->dlp_next = bufs;
1385                         bufs = pair;
1386
1387                         bcopy(data_str->d_buf, pair->dlp_str, data_str->d_size);
1388                         data_str->d_buf = pair->dlp_str;
1389                         data_str->d_size += len;
1390                         (void) elf_flagdata(data_str, ELF_C_SET, ELF_F_DIRTY);
1391
1392                         shdr_str.sh_size += len;
1393                         (void) gelf_update_shdr(scn_str, &shdr_str);
1394
1395                         bcopy(data_sym->d_buf, pair->dlp_sym, data_sym->d_size);
1396                         data_sym->d_buf = pair->dlp_sym;
1397                         data_sym->d_size += nsym * symsize;
1398                         (void) elf_flagdata(data_sym, ELF_C_SET, ELF_F_DIRTY);
1399
1400                         shdr_sym.sh_size += nsym * symsize;
1401                         (void) gelf_update_shdr(scn_sym, &shdr_sym);
1402
1403                         nsym += isym;
1404                 } else {
1405                         dt_strtab_destroy(strtab);
1406                 }
1407
1408                 /*
1409                  * Now that the tables have been allocated, perform the
1410                  * modifications described above.
1411                  */
1412                 for (i = 0; i < shdr_rel.sh_size / shdr_rel.sh_entsize; i++) {
1413
1414                         if (shdr_rel.sh_type == SHT_RELA) {
1415                                 if (gelf_getrela(data_rel, i, &rela) == NULL)
1416                                         continue;
1417                         } else {
1418                                 GElf_Rel rel;
1419                                 if (gelf_getrel(data_rel, i, &rel) == NULL)
1420                                         continue;
1421                                 rela.r_offset = rel.r_offset;
1422                                 rela.r_info = rel.r_info;
1423                                 rela.r_addend = 0;
1424                         }
1425
1426                         ndx = GELF_R_SYM(rela.r_info);
1427
1428                         if (gelf_getsym(data_sym, ndx, &rsym) == NULL ||
1429                             rsym.st_name > data_str->d_size)
1430                                 goto err;
1431
1432                         s = (char *)data_str->d_buf + rsym.st_name;
1433
1434                         if (strncmp(s, dt_prefix, sizeof (dt_prefix) - 1) != 0)
1435                                 continue;
1436
1437                         s += sizeof (dt_prefix) - 1;
1438
1439                         /*
1440                          * Check to see if this is an 'is-enabled' check as
1441                          * opposed to a normal probe.
1442                          */
1443                         if (strncmp(s, dt_enabled,
1444                             sizeof (dt_enabled) - 1) == 0) {
1445                                 s += sizeof (dt_enabled) - 1;
1446                                 eprobe = 1;
1447                                 *eprobesp = 1;
1448                                 dt_dprintf("is-enabled probe\n");
1449                         } else {
1450                                 eprobe = 0;
1451                                 dt_dprintf("normal probe\n");
1452                         }
1453
1454                         if (*s++ != '_')
1455                                 goto err;
1456
1457                         if ((p = strstr(s, "___")) == NULL ||
1458                             p - s >= sizeof (pname))
1459                                 goto err;
1460
1461                         bcopy(s, pname, p - s);
1462                         pname[p - s] = '\0';
1463
1464                         p = strhyphenate(p + 3); /* strlen("___") */
1465
1466                         if (dt_symtab_lookup(data_sym, isym, rela.r_offset,
1467                             shdr_rel.sh_info, &fsym) != 0)
1468                                 goto err;
1469
1470                         if (fsym.st_name > data_str->d_size)
1471                                 goto err;
1472
1473                         assert(GELF_ST_TYPE(fsym.st_info) == STT_FUNC);
1474
1475                         /*
1476                          * If a NULL relocation name is passed to
1477                          * dt_probe_define(), the function name is used for the
1478                          * relocation. The relocation needs to use a mangled
1479                          * name if the symbol is locally scoped; the function
1480                          * name may need to change if we've found the global
1481                          * alias for the locally scoped symbol (we prefer
1482                          * global symbols to locals in dt_symtab_lookup()).
1483                          */
1484                         s = (char *)data_str->d_buf + fsym.st_name;
1485                         r = NULL;
1486
1487                         if (GELF_ST_BIND(fsym.st_info) == STB_LOCAL) {
1488                                 dsym = fsym;
1489                                 dsym.st_name = istr;
1490                                 dsym.st_info = GELF_ST_INFO(STB_GLOBAL,
1491                                     STT_FUNC);
1492                                 dsym.st_other =
1493                                     ELF64_ST_VISIBILITY(STV_ELIMINATE);
1494                                 (void) gelf_update_sym(data_sym, isym, &dsym);
1495
1496                                 r = (char *)data_str->d_buf + istr;
1497                                 istr += 1 + sprintf(r, dt_symfmt,
1498                                     dt_symprefix, objkey, s);
1499                                 isym++;
1500                                 assert(isym <= nsym);
1501
1502                         } else if (strncmp(s, dt_symprefix,
1503                             strlen(dt_symprefix)) == 0) {
1504                                 r = s;
1505                                 if ((s = strchr(s, '.')) == NULL)
1506                                         goto err;
1507                                 s++;
1508                         }
1509
1510                         if ((pvp = dt_provider_lookup(dtp, pname)) == NULL) {
1511                                 return (dt_link_error(dtp, elf, fd, bufs,
1512                                     "no such provider %s", pname));
1513                         }
1514
1515                         if ((prp = dt_probe_lookup(pvp, p)) == NULL) {
1516                                 return (dt_link_error(dtp, elf, fd, bufs,
1517                                     "no such probe %s", p));
1518                         }
1519
1520                         assert(fsym.st_value <= rela.r_offset);
1521
1522                         off = rela.r_offset - fsym.st_value;
1523                         if (dt_modtext(dtp, data_tgt->d_buf, eprobe,
1524                             &rela, &off) != 0)
1525                                 goto err;
1526
1527                         if (dt_probe_define(pvp, prp, s, r, off, eprobe) != 0) {
1528                                 return (dt_link_error(dtp, elf, fd, bufs,
1529                                     "failed to allocate space for probe"));
1530                         }
1531 #if !defined(sun)
1532                         /*
1533                          * Our linker doesn't understand the SUNW_IGNORE ndx and
1534                          * will try to use this relocation when we build the
1535                          * final executable. Since we are done processing this
1536                          * relocation, mark it as inexistant and let libelf
1537                          * remove it from the file.
1538                          * If this wasn't done, we would have garbage added to
1539                          * the executable file as the symbol is going to be
1540                          * change from UND to ABS.
1541                          */
1542                         rela.r_offset = 0;
1543                         rela.r_info  = 0;
1544                         rela.r_addend = 0;
1545                         (void) gelf_update_rela(data_rel, i, &rela);
1546 #endif
1547
1548                         mod = 1;
1549                         (void) elf_flagdata(data_tgt, ELF_C_SET, ELF_F_DIRTY);
1550
1551                         /*
1552                          * This symbol may already have been marked to
1553                          * be ignored by another relocation referencing
1554                          * the same symbol or if this object file has
1555                          * already been processed by an earlier link
1556                          * invocation.
1557                          */
1558 #if !defined(sun)
1559 #define SHN_SUNW_IGNORE SHN_ABS
1560 #endif
1561                         if (rsym.st_shndx != SHN_SUNW_IGNORE) {
1562                                 rsym.st_shndx = SHN_SUNW_IGNORE;
1563                                 (void) gelf_update_sym(data_sym, ndx, &rsym);
1564                         }
1565                 }
1566         }
1567
1568         if (mod && elf_update(elf, ELF_C_WRITE) == -1)
1569                 goto err;
1570
1571         (void) elf_end(elf);
1572         (void) close(fd);
1573
1574 #if !defined(sun)
1575         if (nsym > 0)
1576 #endif
1577         while ((pair = bufs) != NULL) {
1578                 bufs = pair->dlp_next;
1579                 dt_free(dtp, pair->dlp_str);
1580                 dt_free(dtp, pair->dlp_sym);
1581                 dt_free(dtp, pair);
1582         }
1583
1584         return (0);
1585
1586 err:
1587         return (dt_link_error(dtp, elf, fd, bufs,
1588             "an error was encountered while processing %s", obj));
1589 }
1590
1591 int
1592 dtrace_program_link(dtrace_hdl_t *dtp, dtrace_prog_t *pgp, uint_t dflags,
1593     const char *file, int objc, char *const objv[])
1594 {
1595 #if !defined(sun)
1596         char tfile[PATH_MAX];
1597         Elf *e;
1598         Elf_Scn *scn;
1599         Elf_Data *data;
1600         GElf_Shdr shdr;
1601         int efd;
1602         size_t stridx;
1603         unsigned char *buf;
1604         char *s;
1605         int loc;
1606         GElf_Ehdr ehdr;
1607         Elf_Scn *scn0;
1608         GElf_Shdr shdr0;
1609         uint64_t off, rc;
1610 #endif
1611         char drti[PATH_MAX];
1612         dof_hdr_t *dof;
1613         int fd, status, i, cur;
1614         char *cmd, tmp;
1615         size_t len;
1616         int eprobes = 0, ret = 0;
1617
1618 #if !defined(sun)
1619         if (access(file, R_OK) == 0) {
1620                 fprintf(stderr, "dtrace: target object (%s) already exists. "
1621                     "Please remove the target\ndtrace: object and rebuild all "
1622                     "the source objects if you wish to run the DTrace\n"
1623                     "dtrace: linking process again\n", file);
1624                 /*
1625                  * Several build infrastructures run DTrace twice (e.g.
1626                  * postgres) and we don't want the build to fail. Return
1627                  * 0 here since this isn't really a fatal error.
1628                  */
1629                 return (0);
1630         }
1631         /* XXX Should get a temp file name here. */
1632         snprintf(tfile, sizeof(tfile), "%s.tmp", file);
1633 #endif
1634
1635         /*
1636          * A NULL program indicates a special use in which we just link
1637          * together a bunch of object files specified in objv and then
1638          * unlink(2) those object files.
1639          */
1640         if (pgp == NULL) {
1641                 const char *fmt = "%s -o %s -r";
1642
1643                 len = snprintf(&tmp, 1, fmt, dtp->dt_ld_path, file) + 1;
1644
1645                 for (i = 0; i < objc; i++)
1646                         len += strlen(objv[i]) + 1;
1647
1648                 cmd = alloca(len);
1649
1650                 cur = snprintf(cmd, len, fmt, dtp->dt_ld_path, file);
1651
1652                 for (i = 0; i < objc; i++)
1653                         cur += snprintf(cmd + cur, len - cur, " %s", objv[i]);
1654
1655                 if ((status = system(cmd)) == -1) {
1656                         return (dt_link_error(dtp, NULL, -1, NULL,
1657                             "failed to run %s: %s", dtp->dt_ld_path,
1658                             strerror(errno)));
1659                 }
1660
1661                 if (WIFSIGNALED(status)) {
1662                         return (dt_link_error(dtp, NULL, -1, NULL,
1663                             "failed to link %s: %s failed due to signal %d",
1664                             file, dtp->dt_ld_path, WTERMSIG(status)));
1665                 }
1666
1667                 if (WEXITSTATUS(status) != 0) {
1668                         return (dt_link_error(dtp, NULL, -1, NULL,
1669                             "failed to link %s: %s exited with status %d\n",
1670                             file, dtp->dt_ld_path, WEXITSTATUS(status)));
1671                 }
1672
1673                 for (i = 0; i < objc; i++) {
1674                         if (strcmp(objv[i], file) != 0)
1675                                 (void) unlink(objv[i]);
1676                 }
1677
1678                 return (0);
1679         }
1680
1681         for (i = 0; i < objc; i++) {
1682                 if (process_obj(dtp, objv[i], &eprobes) != 0)
1683                         return (-1); /* errno is set for us */
1684         }
1685
1686         /*
1687          * If there are is-enabled probes then we need to force use of DOF
1688          * version 2.
1689          */
1690         if (eprobes && pgp->dp_dofversion < DOF_VERSION_2)
1691                 pgp->dp_dofversion = DOF_VERSION_2;
1692
1693         if ((dof = dtrace_dof_create(dtp, pgp, dflags)) == NULL)
1694                 return (-1); /* errno is set for us */
1695
1696 #if defined(sun)
1697         /*
1698          * Create a temporary file and then unlink it if we're going to
1699          * combine it with drti.o later.  We can still refer to it in child
1700          * processes as /dev/fd/<fd>.
1701          */
1702         if ((fd = open64(file, O_RDWR | O_CREAT | O_TRUNC, 0666)) == -1) {
1703                 return (dt_link_error(dtp, NULL, -1, NULL,
1704                     "failed to open %s: %s", file, strerror(errno)));
1705         }
1706 #else
1707         if ((fd = open(tfile, O_RDWR | O_CREAT | O_TRUNC, 0666)) == -1)
1708                 return (dt_link_error(dtp, NULL, -1, NULL,
1709                     "failed to open %s: %s", tfile, strerror(errno)));
1710 #endif
1711
1712         /*
1713          * If -xlinktype=DOF has been selected, just write out the DOF.
1714          * Otherwise proceed to the default of generating and linking ELF.
1715          */
1716         switch (dtp->dt_linktype) {
1717         case DT_LTYP_DOF:
1718                 if (dt_write(dtp, fd, dof, dof->dofh_filesz) < dof->dofh_filesz)
1719                         ret = errno;
1720
1721                 if (close(fd) != 0 && ret == 0)
1722                         ret = errno;
1723
1724                 if (ret != 0) {
1725                         return (dt_link_error(dtp, NULL, -1, NULL,
1726                             "failed to write %s: %s", file, strerror(ret)));
1727                 }
1728
1729                 return (0);
1730
1731         case DT_LTYP_ELF:
1732                 break; /* fall through to the rest of dtrace_program_link() */
1733
1734         default:
1735                 return (dt_link_error(dtp, NULL, -1, NULL,
1736                     "invalid link type %u\n", dtp->dt_linktype));
1737         }
1738
1739
1740 #if defined(sun)
1741         if (!dtp->dt_lazyload)
1742                 (void) unlink(file);
1743 #endif
1744
1745 #if defined(sun)
1746         if (dtp->dt_oflags & DTRACE_O_LP64)
1747                 status = dump_elf64(dtp, dof, fd);
1748         else
1749                 status = dump_elf32(dtp, dof, fd);
1750
1751         if (status != 0 || lseek(fd, 0, SEEK_SET) != 0) {
1752 #else
1753         /* We don't write the ELF header, just the DOF section */
1754         if (dt_write(dtp, fd, dof, dof->dofh_filesz) < dof->dofh_filesz) {
1755 #endif
1756                 return (dt_link_error(dtp, NULL, -1, NULL,
1757                     "failed to write %s: %s", file, strerror(errno)));
1758         }
1759
1760         if (!dtp->dt_lazyload) {
1761 #if defined(sun)
1762                 const char *fmt = "%s -o %s -r -Blocal -Breduce /dev/fd/%d %s";
1763
1764                 if (dtp->dt_oflags & DTRACE_O_LP64) {
1765                         (void) snprintf(drti, sizeof (drti),
1766                             "%s/64/drti.o", _dtrace_libdir);
1767                 } else {
1768                         (void) snprintf(drti, sizeof (drti),
1769                             "%s/drti.o", _dtrace_libdir);
1770                 }
1771
1772                 len = snprintf(&tmp, 1, fmt, dtp->dt_ld_path, file, fd,
1773                     drti) + 1;
1774
1775                 cmd = alloca(len);
1776
1777                 (void) snprintf(cmd, len, fmt, dtp->dt_ld_path, file, fd, drti);
1778 #else
1779                 const char *fmt = "%s -o %s -r %s";
1780
1781 #if defined(__amd64__)
1782                 /*
1783                  * Arches which default to 64-bit need to explicitly use
1784                  * the 32-bit library path.
1785                  */
1786                 int use_32 = !(dtp->dt_oflags & DTRACE_O_LP64);
1787 #else
1788                 /*
1789                  * Arches which are 32-bit only just use the normal
1790                  * library path.
1791                  */
1792                 int use_32 = 0;
1793 #endif
1794
1795                 (void) snprintf(drti, sizeof (drti), "/usr/lib%s/dtrace/drti.o",
1796                     use_32 ? "32":"");
1797
1798                 len = snprintf(&tmp, 1, fmt, dtp->dt_ld_path, file, tfile,
1799                     drti) + 1;
1800
1801 #if !defined(sun)
1802                 len *= 2;
1803 #endif
1804                 cmd = alloca(len);
1805
1806                 (void) snprintf(cmd, len, fmt, dtp->dt_ld_path, file,
1807                     drti);
1808 #endif
1809                 if ((status = system(cmd)) == -1) {
1810                         ret = dt_link_error(dtp, NULL, -1, NULL,
1811                             "failed to run %s: %s", dtp->dt_ld_path,
1812                             strerror(errno));
1813                         goto done;
1814                 }
1815
1816                 if (WIFSIGNALED(status)) {
1817                         ret = dt_link_error(dtp, NULL, -1, NULL,
1818                             "failed to link %s: %s failed due to signal %d",
1819                             file, dtp->dt_ld_path, WTERMSIG(status));
1820                         goto done;
1821                 }
1822
1823                 if (WEXITSTATUS(status) != 0) {
1824                         ret = dt_link_error(dtp, NULL, -1, NULL,
1825                             "failed to link %s: %s exited with status %d\n",
1826                             file, dtp->dt_ld_path, WEXITSTATUS(status));
1827                         goto done;
1828                 }
1829 #if !defined(sun)
1830 #define BROKEN_LIBELF
1831                 /*
1832                  * FreeBSD's ld(1) is not instructed to interpret and add
1833                  * correctly the SUNW_dof section present in tfile.
1834                  * We use libelf to add this section manually and hope the next
1835                  * ld invocation won't remove it.
1836                  */
1837                 elf_version(EV_CURRENT);
1838                 if ((efd = open(file, O_RDWR, 0)) < 0) {
1839                         ret = dt_link_error(dtp, NULL, -1, NULL,
1840                             "failed to open file %s: %s",
1841                             file, strerror(errno));
1842                         goto done;
1843                 }
1844                 if ((e = elf_begin(efd, ELF_C_RDWR, NULL)) == NULL) {
1845                         close(efd);
1846                         ret = dt_link_error(dtp, NULL, -1, NULL,
1847                             "failed to open elf file: %s",
1848                             elf_errmsg(elf_errno()));
1849                         goto done;
1850                 }
1851                 /*
1852                  * Add the string '.SUWN_dof' to the shstrtab section.
1853                  */
1854 #ifdef BROKEN_LIBELF
1855                 elf_flagelf(e, ELF_C_SET, ELF_F_LAYOUT);
1856 #endif
1857                 elf_getshdrstrndx(e, &stridx);
1858                 scn = elf_getscn(e, stridx);
1859                 gelf_getshdr(scn, &shdr);
1860                 data = elf_newdata(scn);
1861                 data->d_off = shdr.sh_size;
1862                 data->d_buf = ".SUNW_dof";
1863                 data->d_size = 10;
1864                 data->d_type = ELF_T_BYTE;
1865                 loc = shdr.sh_size;
1866                 shdr.sh_size += data->d_size;
1867                 gelf_update_shdr(scn, &shdr);
1868 #ifdef BROKEN_LIBELF
1869                 off = shdr.sh_offset;
1870                 rc = shdr.sh_offset + shdr.sh_size;
1871                 gelf_getehdr(e, &ehdr);
1872                 if (ehdr.e_shoff > off) {
1873                         off = ehdr.e_shoff + ehdr.e_shnum * ehdr.e_shentsize;
1874                         rc = roundup(rc, 8);
1875                         ehdr.e_shoff = rc;
1876                         gelf_update_ehdr(e, &ehdr);
1877                         rc += ehdr.e_shnum * ehdr.e_shentsize;
1878                 }
1879                 for (;;) {
1880                         scn0 = NULL;
1881                         scn = NULL;
1882                         while ((scn = elf_nextscn(e, scn)) != NULL) {
1883                                 gelf_getshdr(scn, &shdr);
1884                                 if (shdr.sh_type == SHT_NOBITS ||
1885                                     shdr.sh_offset < off)
1886                                         continue;
1887                                 /* Find the immediately adjcent section. */
1888                                 if (scn0 == NULL ||
1889                                     shdr.sh_offset < shdr0.sh_offset) {
1890                                         scn0 = scn;
1891                                         gelf_getshdr(scn0, &shdr0);
1892                                 }
1893                         }
1894                         if (scn0 == NULL)
1895                                 break;
1896                         /* Load section data to work around another bug */
1897                         elf_getdata(scn0, NULL);
1898                         /* Update section header, assure section alignment */
1899                         off = shdr0.sh_offset + shdr0.sh_size;
1900                         rc = roundup(rc, shdr0.sh_addralign);
1901                         shdr0.sh_offset = rc;
1902                         gelf_update_shdr(scn0, &shdr0);
1903                         rc += shdr0.sh_size;
1904                 }
1905                 if (elf_update(e, ELF_C_WRITE) < 0) {
1906                         ret = dt_link_error(dtp, NULL, -1, NULL,
1907                             "failed to add append the shstrtab section: %s",
1908                             elf_errmsg(elf_errno()));
1909                         elf_end(e);
1910                         close(efd);
1911                         goto done;
1912                 }
1913                 elf_end(e);
1914                 e = elf_begin(efd, ELF_C_RDWR, NULL);
1915 #endif
1916                 /*
1917                  * Construct the .SUNW_dof section.
1918                  */
1919                 scn = elf_newscn(e);
1920                 data = elf_newdata(scn);
1921                 buf = mmap(NULL, dof->dofh_filesz, PROT_READ, MAP_SHARED,
1922                     fd, 0);
1923                 if (buf == MAP_FAILED) {
1924                         ret = dt_link_error(dtp, NULL, -1, NULL,
1925                             "failed to mmap buffer %s", strerror(errno));
1926                         elf_end(e);
1927                         close(efd);
1928                         goto done;
1929                 }
1930                 data->d_buf = buf;
1931                 data->d_align = 4;
1932                 data->d_size = dof->dofh_filesz;
1933                 data->d_version = EV_CURRENT;
1934                 gelf_getshdr(scn, &shdr);
1935                 shdr.sh_name = loc;
1936                 shdr.sh_flags = SHF_ALLOC;
1937                 /*
1938                  * Actually this should be SHT_SUNW_dof, but FreeBSD's ld(1)
1939                  * will remove this 'unknown' section when we try to create an
1940                  * executable using the object we are modifying, so we stop
1941                  * playing by the rules and use SHT_PROGBITS.
1942                  * Also, note that our drti has modifications to handle this.
1943                  */
1944                 shdr.sh_type = SHT_PROGBITS;
1945                 shdr.sh_addralign = 4;
1946                 gelf_update_shdr(scn, &shdr);
1947                 if (elf_update(e, ELF_C_WRITE) < 0) {
1948                         ret = dt_link_error(dtp, NULL, -1, NULL,
1949                             "failed to add the SUNW_dof section: %s",
1950                             elf_errmsg(elf_errno()));
1951                         munmap(buf, dof->dofh_filesz);
1952                         elf_end(e);
1953                         close(efd);
1954                         goto done;
1955                 }
1956                 munmap(buf, dof->dofh_filesz);
1957                 elf_end(e);
1958                 close(efd);
1959 #endif
1960                 (void) close(fd); /* release temporary file */
1961         } else {
1962                 (void) close(fd);
1963         }
1964
1965 done:
1966         dtrace_dof_destroy(dtp, dof);
1967
1968 #if !defined(sun)
1969         unlink(tfile);
1970 #endif
1971         return (ret);
1972 }