sys/dev/sound/pcm/feeder_rate.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2005-2009 Ariff Abdullah <ariff@FreeBSD.org>
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 /*
  30  * feeder_rate: (Codename: Z Resampler), which means any effort to create
  31  *              future replacement for this resampler are simply absurd unless
  32  *              the world decide to add new alphabet after Z.
  33  *
  34  * FreeBSD bandlimited sinc interpolator, technically based on
  35  * "Digital Audio Resampling" by Julius O. Smith III
  36  *  - http://ccrma.stanford.edu/~jos/resample/
  37  *
  38  * The Good:
  39  * + all out fixed point integer operations, no soft-float or anything like
  40  *   that.
  41  * + classic polyphase converters with high quality coefficient's polynomial
  42  *   interpolators.
  43  * + fast, faster, or the fastest of its kind.
  44  * + compile time configurable.
  45  * + etc etc..
  46  *
  47  * The Bad:
  48  * - The z, z_, and Z_ . Due to mental block (or maybe just 0x7a69), I
  49  *   couldn't think of anything simpler than that (feeder_rate_xxx is just
  50  *   too long). Expect possible clashes with other zitizens (any?).
  51  */
  52
  53 #ifdef _KERNEL
  54 #ifdef HAVE_KERNEL_OPTION_HEADERS
  55 #include "opt_snd.h"
  56 #endif
  57 #include <dev/sound/pcm/sound.h>
  58 #include <dev/sound/pcm/pcm.h>
  59 #include "feeder_if.h"
  60
  61 #define SND_USE_FXDIV
  62 #include "snd_fxdiv_gen.h"
  63
  64 SND_DECLARE_FILE("$FreeBSD$");
  65 #endif
  66
  67 #include "feeder_rate_gen.h"
  68
  69 #if !defined(_KERNEL) && defined(SND_DIAGNOSTIC)
  70 #undef Z_DIAGNOSTIC
  71 #define Z_DIAGNOSTIC            1
  72 #elif defined(_KERNEL)
  73 #undef Z_DIAGNOSTIC
  74 #endif
  75
  76 #ifndef Z_QUALITY_DEFAULT
  77 #define Z_QUALITY_DEFAULT       Z_QUALITY_LINEAR
  78 #endif
  79
  80 #define Z_RESERVOIR             2048
  81 #define Z_RESERVOIR_MAX         131072
  82
  83 #define Z_SINC_MAX              0x3fffff
  84 #define Z_SINC_DOWNMAX          48              /* 384000 / 8000 */
  85
  86 #ifdef _KERNEL
  87 #define Z_POLYPHASE_MAX         183040          /* 286 taps, 640 phases */
  88 #else
  89 #define Z_POLYPHASE_MAX         1464320         /* 286 taps, 5120 phases */
  90 #endif
  91
  92 #define Z_RATE_DEFAULT          48000
  93
  94 #define Z_RATE_MIN              FEEDRATE_RATEMIN
  95 #define Z_RATE_MAX              FEEDRATE_RATEMAX
  96 #define Z_ROUNDHZ               FEEDRATE_ROUNDHZ
  97 #define Z_ROUNDHZ_MIN           FEEDRATE_ROUNDHZ_MIN
  98 #define Z_ROUNDHZ_MAX           FEEDRATE_ROUNDHZ_MAX
  99
 100 #define Z_RATE_SRC              FEEDRATE_SRC
 101 #define Z_RATE_DST              FEEDRATE_DST
 102 #define Z_RATE_QUALITY          FEEDRATE_QUALITY
 103 #define Z_RATE_CHANNELS         FEEDRATE_CHANNELS
 104
 105 #define Z_PARANOID              1
 106
 107 #define Z_MULTIFORMAT           1
 108
 109 #ifdef _KERNEL
 110 #undef Z_USE_ALPHADRIFT
 111 #define Z_USE_ALPHADRIFT        1
 112 #endif
 113
 114 #define Z_FACTOR_MIN            1
 115 #define Z_FACTOR_MAX            Z_MASK
 116 #define Z_FACTOR_SAFE(v)        (!((v) < Z_FACTOR_MIN || (v) > Z_FACTOR_MAX))
 117
 118 struct z_info;
 119
 120 typedef void (*z_resampler_t)(struct z_info *, uint8_t *);
 121
 122 struct z_info {
 123         int32_t rsrc, rdst;     /* original source / destination rates */
 124         int32_t src, dst;       /* rounded source / destination rates */
 125         int32_t channels;       /* total channels */
 126         int32_t bps;            /* bytes-per-sample */
 127         int32_t quality;        /* resampling quality */
 128
 129         int32_t z_gx, z_gy;     /* interpolation / decimation ratio */
 130         int32_t z_alpha;        /* output sample time phase / drift */
 131         uint8_t *z_delay;       /* FIR delay line / linear buffer */
 132         int32_t *z_coeff;       /* FIR coefficients */
 133         int32_t *z_dcoeff;      /* FIR coefficients differences */
 134         int32_t *z_pcoeff;      /* FIR polyphase coefficients */
 135         int32_t z_scale;        /* output scaling */
 136         int32_t z_dx;           /* input sample drift increment */
 137         int32_t z_dy;           /* output sample drift increment */
 138 #ifdef Z_USE_ALPHADRIFT
 139         int32_t z_alphadrift;   /* alpha drift rate */
 140         int32_t z_startdrift;   /* buffer start position drift rate */
 141 #endif
 142         int32_t z_mask;         /* delay line full length mask */
 143         int32_t z_size;         /* half width of FIR taps */
 144         int32_t z_full;         /* full size of delay line */
 145         int32_t z_alloc;        /* largest allocated full size of delay line */
 146         int32_t z_start;        /* buffer processing start position */
 147         int32_t z_pos;          /* current position for the next feed */
 148 #ifdef Z_DIAGNOSTIC
 149         uint32_t z_cycle;       /* output cycle, purely for statistical */
 150 #endif
 151         int32_t z_maxfeed;      /* maximum feed to avoid 32bit overflow */
 152
 153         z_resampler_t z_resample;
 154 };
 155
 156 int feeder_rate_min = Z_RATE_MIN;
 157 int feeder_rate_max = Z_RATE_MAX;
 158 int feeder_rate_round = Z_ROUNDHZ;
 159 int feeder_rate_quality = Z_QUALITY_DEFAULT;
 160
 161 static int feeder_rate_polyphase_max = Z_POLYPHASE_MAX;
 162
 163 #ifdef _KERNEL
 164 static char feeder_rate_presets[] = FEEDER_RATE_PRESETS;
 165 SYSCTL_STRING(_hw_snd, OID_AUTO, feeder_rate_presets, CTLFLAG_RD,
 166     &feeder_rate_presets, 0, "compile-time rate presets");
 167 SYSCTL_INT(_hw_snd, OID_AUTO, feeder_rate_polyphase_max, CTLFLAG_RWTUN,
 168     &feeder_rate_polyphase_max, 0, "maximum allowable polyphase entries");
 169
 170 static int
 171 sysctl_hw_snd_feeder_rate_min(SYSCTL_HANDLER_ARGS)
 172 {
 173         int err, val;
 174
 175         val = feeder_rate_min;
 176         err = sysctl_handle_int(oidp, &val, 0, req);
 177
 178         if (err != 0 || req->newptr == NULL || val == feeder_rate_min)
 179                 return (err);
 180
 181         if (!(Z_FACTOR_SAFE(val) && val < feeder_rate_max))
 182                 return (EINVAL);
 183
 184         feeder_rate_min = val;
 185
 186         return (0);
 187 }
 188 SYSCTL_PROC(_hw_snd, OID_AUTO, feeder_rate_min,
 189     CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_NEEDGIANT, 0, sizeof(int),
 190     sysctl_hw_snd_feeder_rate_min, "I",
 191     "minimum allowable rate");
 192
 193 static int
 194 sysctl_hw_snd_feeder_rate_max(SYSCTL_HANDLER_ARGS)
 195 {
 196         int err, val;
 197
 198         val = feeder_rate_max;
 199         err = sysctl_handle_int(oidp, &val, 0, req);
 200
 201         if (err != 0 || req->newptr == NULL || val == feeder_rate_max)
 202                 return (err);
 203
 204         if (!(Z_FACTOR_SAFE(val) && val > feeder_rate_min))
 205                 return (EINVAL);
 206
 207         feeder_rate_max = val;
 208
 209         return (0);
 210 }
 211 SYSCTL_PROC(_hw_snd, OID_AUTO, feeder_rate_max,
 212     CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_NEEDGIANT, 0, sizeof(int),
 213     sysctl_hw_snd_feeder_rate_max, "I",
 214     "maximum allowable rate");
 215
 216 static int
 217 sysctl_hw_snd_feeder_rate_round(SYSCTL_HANDLER_ARGS)
 218 {
 219         int err, val;
 220
 221         val = feeder_rate_round;
 222         err = sysctl_handle_int(oidp, &val, 0, req);
 223
 224         if (err != 0 || req->newptr == NULL || val == feeder_rate_round)
 225                 return (err);
 226
 227         if (val < Z_ROUNDHZ_MIN || val > Z_ROUNDHZ_MAX)
 228                 return (EINVAL);
 229
 230         feeder_rate_round = val - (val % Z_ROUNDHZ);
 231
 232         return (0);
 233 }
 234 SYSCTL_PROC(_hw_snd, OID_AUTO, feeder_rate_round,
 235     CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_NEEDGIANT, 0, sizeof(int),
 236     sysctl_hw_snd_feeder_rate_round, "I",
 237     "sample rate converter rounding threshold");
 238
 239 static int
 240 sysctl_hw_snd_feeder_rate_quality(SYSCTL_HANDLER_ARGS)
 241 {
 242         struct snddev_info *d;
 243         struct pcm_channel *c;
 244         struct pcm_feeder *f;
 245         int i, err, val;
 246
 247         val = feeder_rate_quality;
 248         err = sysctl_handle_int(oidp, &val, 0, req);
 249
 250         if (err != 0 || req->newptr == NULL || val == feeder_rate_quality)
 251                 return (err);
 252
 253         if (val < Z_QUALITY_MIN || val > Z_QUALITY_MAX)
 254                 return (EINVAL);
 255
 256         feeder_rate_quality = val;
 257
 258         /*
 259          * Traverse all available channels on each device and try to
 260          * set resampler quality if and only if it is exist as
 261          * part of feeder chains and the channel is idle.
 262          */
 263         for (i = 0; pcm_devclass != NULL &&
 264             i < devclass_get_maxunit(pcm_devclass); i++) {
 265                 d = devclass_get_softc(pcm_devclass, i);
 266                 if (!PCM_REGISTERED(d))
 267                         continue;
 268                 PCM_LOCK(d);
 269                 PCM_WAIT(d);
 270                 PCM_ACQUIRE(d);
 271                 CHN_FOREACH(c, d, channels.pcm) {
 272                         CHN_LOCK(c);
 273                         f = chn_findfeeder(c, FEEDER_RATE);
 274                         if (f == NULL || f->data == NULL || CHN_STARTED(c)) {
 275                                 CHN_UNLOCK(c);
 276                                 continue;
 277                         }
 278                         (void)FEEDER_SET(f, FEEDRATE_QUALITY, val);
 279                         CHN_UNLOCK(c);
 280                 }
 281                 PCM_RELEASE(d);
 282                 PCM_UNLOCK(d);
 283         }
 284
 285         return (0);
 286 }
 287 SYSCTL_PROC(_hw_snd, OID_AUTO, feeder_rate_quality,
 288     CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_NEEDGIANT, 0, sizeof(int),
 289     sysctl_hw_snd_feeder_rate_quality, "I",
 290     "sample rate converter quality ("__XSTRING(Z_QUALITY_MIN)"=low .. "
 291     __XSTRING(Z_QUALITY_MAX)"=high)");
 292 #endif  /* _KERNEL */
 293
 294 /*
 295  * Resampler type.
 296  */
 297 #define Z_IS_ZOH(i)             ((i)->quality == Z_QUALITY_ZOH)
 298 #define Z_IS_LINEAR(i)          ((i)->quality == Z_QUALITY_LINEAR)
 299 #define Z_IS_SINC(i)            ((i)->quality > Z_QUALITY_LINEAR)
 300
 301 /*
 302  * Macroses for accurate sample time drift calculations.
 303  *
 304  * gy2gx : given the amount of output, return the _exact_ required amount of
 305  *         input.
 306  * gx2gy : given the amount of input, return the _maximum_ amount of output
 307  *         that will be generated.
 308  * drift : given the amount of input and output, return the elapsed
 309  *         sample-time.
 310  */
 311 #define _Z_GCAST(x)             ((uint64_t)(x))
 312
 313 #if defined(__GNUCLIKE_ASM) && defined(__i386__)
 314 /*
 315  * This is where i386 being beaten to a pulp. Fortunately this function is
 316  * rarely being called and if it is, it will decide the best (hopefully)
 317  * fastest way to do the division. If we can ensure that everything is dword
 318  * aligned, letting the compiler to call udivdi3 to do the division can be
 319  * faster compared to this.
 320  *
 321  * amd64 is the clear winner here, no question about it.
 322  */
 323 static __inline uint32_t
 324 Z_DIV(uint64_t v, uint32_t d)
 325 {
 326         uint32_t hi, lo, quo, rem;
 327
 328         hi = v >> 32;
 329         lo = v & 0xffffffff;
 330
 331         /*
 332          * As much as we can, try to avoid long division like a plague.
 333          */
 334         if (hi == 0)
 335                 quo = lo / d;
 336         else
 337                 __asm("divl %2"
 338                     : "=a" (quo), "=d" (rem)
 339                     : "r" (d), "0" (lo), "1" (hi));
 340
 341         return (quo);
 342 }
 343 #else
 344 #define Z_DIV(x, y)             ((x) / (y))
 345 #endif
 346
 347 #define _Z_GY2GX(i, a, v)                                               \
 348         Z_DIV(((_Z_GCAST((i)->z_gx) * (v)) + ((i)->z_gy - (a) - 1)),    \
 349         (i)->z_gy)
 350
 351 #define _Z_GX2GY(i, a, v)                                               \
 352         Z_DIV(((_Z_GCAST((i)->z_gy) * (v)) + (a)), (i)->z_gx)
 353
 354 #define _Z_DRIFT(i, x, y)                                               \
 355         ((_Z_GCAST((i)->z_gy) * (x)) - (_Z_GCAST((i)->z_gx) * (y)))
 356
 357 #define z_gy2gx(i, v)           _Z_GY2GX(i, (i)->z_alpha, v)
 358 #define z_gx2gy(i, v)           _Z_GX2GY(i, (i)->z_alpha, v)
 359 #define z_drift(i, x, y)        _Z_DRIFT(i, x, y)
 360
 361 /*
 362  * Macroses for SINC coefficients table manipulations.. whatever.
 363  */
 364 #define Z_SINC_COEFF_IDX(i)     ((i)->quality - Z_QUALITY_LINEAR - 1)
 365
 366 #define Z_SINC_LEN(i)                                                   \
 367         ((int32_t)(((uint64_t)z_coeff_tab[Z_SINC_COEFF_IDX(i)].len <<   \
 368             Z_SHIFT) / (i)->z_dy))
 369
 370 #define Z_SINC_BASE_LEN(i)                                              \
 371         ((z_coeff_tab[Z_SINC_COEFF_IDX(i)].len - 1) >> (Z_DRIFT_SHIFT - 1))
 372
 373 /*
 374  * Macroses for linear delay buffer operations. Alignment is not
 375  * really necessary since we're not using true circular buffer, but it
 376  * will help us guard against possible trespasser. To be honest,
 377  * the linear block operations does not need guarding at all due to
 378  * accurate drifting!
 379  */
 380 #define z_align(i, v)           ((v) & (i)->z_mask)
 381 #define z_next(i, o, v)         z_align(i, (o) + (v))
 382 #define z_prev(i, o, v)         z_align(i, (o) - (v))
 383 #define z_fetched(i)            (z_align(i, (i)->z_pos - (i)->z_start) - 1)
 384 #define z_free(i)               ((i)->z_full - (i)->z_pos)
 385
 386 /*
 387  * Macroses for Bla Bla .. :)
 388  */
 389 #define z_copy(src, dst, sz)    (void)memcpy(dst, src, sz)
 390 #define z_feed(...)             FEEDER_FEED(__VA_ARGS__)
 391
 392 static __inline uint32_t
 393 z_min(uint32_t x, uint32_t y)
 394 {
 395
 396         return ((x < y) ? x : y);
 397 }
 398
 399 static int32_t
 400 z_gcd(int32_t x, int32_t y)
 401 {
 402         int32_t w;
 403
 404         while (y != 0) {
 405                 w = x % y;
 406                 x = y;
 407                 y = w;
 408         }
 409
 410         return (x);
 411 }
 412
 413 static int32_t
 414 z_roundpow2(int32_t v)
 415 {
 416         int32_t i;
 417
 418         i = 1;
 419
 420         /*
 421          * Let it overflow at will..
 422          */
 423         while (i > 0 && i < v)
 424                 i <<= 1;
 425
 426         return (i);
 427 }
 428
 429 /*
 430  * Zero Order Hold, the worst of the worst, an insult against quality,
 431  * but super fast.
 432  */
 433 static void
 434 z_feed_zoh(struct z_info *info, uint8_t *dst)
 435 {
 436 #if 0
 437         z_copy(info->z_delay +
 438             (info->z_start * info->channels * info->bps), dst,
 439             info->channels * info->bps);
 440 #else
 441         uint32_t cnt;
 442         uint8_t *src;
 443
 444         cnt = info->channels * info->bps;
 445         src = info->z_delay + (info->z_start * cnt);
 446
 447         /*
 448          * This is a bit faster than doing bcopy() since we're dealing
 449          * with possible unaligned samples.
 450          */
 451         do {
 452                 *dst++ = *src++;
 453         } while (--cnt != 0);
 454 #endif
 455 }
 456
 457 /*
 458  * Linear Interpolation. This at least sounds better (perceptually) and fast,
 459  * but without any proper filtering which means aliasing still exist and
 460  * could become worst with a right sample. Interpolation centered within
 461  * Z_LINEAR_ONE between the present and previous sample and everything is
 462  * done with simple 32bit scaling arithmetic.
 463  */
 464 #define Z_DECLARE_LINEAR(SIGN, BIT, ENDIAN)                                     \
 465 static void                                                                     \
 466 z_feed_linear_##SIGN##BIT##ENDIAN(struct z_info *info, uint8_t *dst)            \
 467 {                                                                               \
 468         int32_t z;                                                              \
 469         intpcm_t x, y;                                                          \
 470         uint32_t ch;                                                            \
 471         uint8_t *sx, *sy;                                                       \
 472                                                                                 \
 473         z = ((uint32_t)info->z_alpha * info->z_dx) >> Z_LINEAR_UNSHIFT;         \
 474                                                                                 \
 475         sx = info->z_delay + (info->z_start * info->channels *                  \
 476             PCM_##BIT##_BPS);                                                   \
 477         sy = sx - (info->channels * PCM_##BIT##_BPS);                           \
 478                                                                                 \
 479         ch = info->channels;                                                    \
 480                                                                                 \
 481         do {                                                                    \
 482                 x = _PCM_READ_##SIGN##BIT##_##ENDIAN(sx);                       \
 483                 y = _PCM_READ_##SIGN##BIT##_##ENDIAN(sy);                       \
 484                 x = Z_LINEAR_INTERPOLATE_##BIT(z, x, y);                        \
 485                 _PCM_WRITE_##SIGN##BIT##_##ENDIAN(dst, x);                      \
 486                 sx += PCM_##BIT##_BPS;                                          \
 487                 sy += PCM_##BIT##_BPS;                                          \
 488                 dst += PCM_##BIT##_BPS;                                         \
 489         } while (--ch != 0);                                                    \
 490 }
 491
 492 /*
 493  * Userland clipping diagnostic check, not enabled in kernel compilation.
 494  * While doing sinc interpolation, unrealistic samples like full scale sine
 495  * wav will clip, but for other things this will not make any noise at all.
 496  * Everybody should learn how to normalized perceived loudness of their own
 497  * music/sounds/samples (hint: ReplayGain).
 498  */
 499 #ifdef Z_DIAGNOSTIC
 500 #define Z_CLIP_CHECK(v, BIT)    do {                                    \
 501         if ((v) > PCM_S##BIT##_MAX) {                                   \
 502                 fprintf(stderr, "Overflow: v=%jd, max=%jd\n",           \
 503                     (intmax_t)(v), (intmax_t)PCM_S##BIT##_MAX);         \
 504         } else if ((v) < PCM_S##BIT##_MIN) {                            \
 505                 fprintf(stderr, "Underflow: v=%jd, min=%jd\n",          \
 506                     (intmax_t)(v), (intmax_t)PCM_S##BIT##_MIN);         \
 507         }                                                               \
 508 } while (0)
 509 #else
 510 #define Z_CLIP_CHECK(...)
 511 #endif
 512
 513 #define Z_CLAMP(v, BIT)                                                 \
 514         (((v) > PCM_S##BIT##_MAX) ? PCM_S##BIT##_MAX :                  \
 515         (((v) < PCM_S##BIT##_MIN) ? PCM_S##BIT##_MIN : (v)))
 516
 517 /*
 518  * Sine Cardinal (SINC) Interpolation. Scaling is done in 64 bit, so
 519  * there's no point to hold the plate any longer. All samples will be
 520  * shifted to a full 32 bit, scaled and restored during write for
 521  * maximum dynamic range (only for downsampling).
 522  */
 523 #define _Z_SINC_ACCUMULATE(SIGN, BIT, ENDIAN, adv)                      \
 524         c += z >> Z_SHIFT;                                              \
 525         z &= Z_MASK;                                                    \
 526         coeff = Z_COEFF_INTERPOLATE(z, z_coeff[c], z_dcoeff[c]);        \
 527         x = _PCM_READ_##SIGN##BIT##_##ENDIAN(p);                        \
 528         v += Z_NORM_##BIT((intpcm64_t)x * coeff);                       \
 529         z += info->z_dy;                                                \
 530         p adv##= info->channels * PCM_##BIT##_BPS
 531
 532 /*
 533  * XXX GCC4 optimization is such a !@#$%, need manual unrolling.
 534  */
 535 #if defined(__GNUC__) && __GNUC__ >= 4
 536 #define Z_SINC_ACCUMULATE(...)  do {                                    \
 537         _Z_SINC_ACCUMULATE(__VA_ARGS__);                                \
 538         _Z_SINC_ACCUMULATE(__VA_ARGS__);                                \
 539 } while (0)
 540 #define Z_SINC_ACCUMULATE_DECR          2
 541 #else
 542 #define Z_SINC_ACCUMULATE(...)  do {                                    \
 543         _Z_SINC_ACCUMULATE(__VA_ARGS__);                                \
 544 } while (0)
 545 #define Z_SINC_ACCUMULATE_DECR          1
 546 #endif
 547
 548 #define Z_DECLARE_SINC(SIGN, BIT, ENDIAN)                                       \
 549 static void                                                                     \
 550 z_feed_sinc_##SIGN##BIT##ENDIAN(struct z_info *info, uint8_t *dst)              \
 551 {                                                                               \
 552         intpcm64_t v;                                                           \
 553         intpcm_t x;                                                             \
 554         uint8_t *p;                                                             \
 555         int32_t coeff, z, *z_coeff, *z_dcoeff;                                  \
 556         uint32_t c, center, ch, i;                                              \
 557                                                                                 \
 558         z_coeff = info->z_coeff;                                                \
 559         z_dcoeff = info->z_dcoeff;                                              \
 560         center = z_prev(info, info->z_start, info->z_size);                     \
 561         ch = info->channels * PCM_##BIT##_BPS;                                  \
 562         dst += ch;                                                              \
 563                                                                                 \
 564         do {                                                                    \
 565                 dst -= PCM_##BIT##_BPS;                                         \
 566                 ch -= PCM_##BIT##_BPS;                                          \
 567                 v = 0;                                                          \
 568                 z = info->z_alpha * info->z_dx;                                 \
 569                 c = 0;                                                          \
 570                 p = info->z_delay + (z_next(info, center, 1) *                  \
 571                     info->channels * PCM_##BIT##_BPS) + ch;                     \
 572                 for (i = info->z_size; i != 0; i -= Z_SINC_ACCUMULATE_DECR)     \
 573                         Z_SINC_ACCUMULATE(SIGN, BIT, ENDIAN, +);                \
 574                 z = info->z_dy - (info->z_alpha * info->z_dx);                  \
 575                 c = 0;                                                          \
 576                 p = info->z_delay + (center * info->channels *                  \
 577                     PCM_##BIT##_BPS) + ch;                                      \
 578                 for (i = info->z_size; i != 0; i -= Z_SINC_ACCUMULATE_DECR)     \
 579                         Z_SINC_ACCUMULATE(SIGN, BIT, ENDIAN, -);                \
 580                 if (info->z_scale != Z_ONE)                                     \
 581                         v = Z_SCALE_##BIT(v, info->z_scale);                    \
 582                 else                                                            \
 583                         v >>= Z_COEFF_SHIFT - Z_GUARD_BIT_##BIT;                \
 584                 Z_CLIP_CHECK(v, BIT);                                           \
 585                 _PCM_WRITE_##SIGN##BIT##_##ENDIAN(dst, Z_CLAMP(v, BIT));        \
 586         } while (ch != 0);                                                      \
 587 }
 588
 589 #define Z_DECLARE_SINC_POLYPHASE(SIGN, BIT, ENDIAN)                             \
 590 static void                                                                     \
 591 z_feed_sinc_polyphase_##SIGN##BIT##ENDIAN(struct z_info *info, uint8_t *dst)    \
 592 {                                                                               \
 593         intpcm64_t v;                                                           \
 594         intpcm_t x;                                                             \
 595         uint8_t *p;                                                             \
 596         int32_t ch, i, start, *z_pcoeff;                                        \
 597                                                                                 \
 598         ch = info->channels * PCM_##BIT##_BPS;                                  \
 599         dst += ch;                                                              \
 600         start = z_prev(info, info->z_start, (info->z_size << 1) - 1) * ch;      \
 601                                                                                 \
 602         do {                                                                    \
 603                 dst -= PCM_##BIT##_BPS;                                         \
 604                 ch -= PCM_##BIT##_BPS;                                          \
 605                 v = 0;                                                          \
 606                 p = info->z_delay + start + ch;                                 \
 607                 z_pcoeff = info->z_pcoeff +                                     \
 608                     ((info->z_alpha * info->z_size) << 1);                      \
 609                 for (i = info->z_size; i != 0; i--) {                           \
 610                         x = _PCM_READ_##SIGN##BIT##_##ENDIAN(p);                \
 611                         v += Z_NORM_##BIT((intpcm64_t)x * *z_pcoeff);           \
 612                         z_pcoeff++;                                             \
 613                         p += info->channels * PCM_##BIT##_BPS;                  \
 614                         x = _PCM_READ_##SIGN##BIT##_##ENDIAN(p);                \
 615                         v += Z_NORM_##BIT((intpcm64_t)x * *z_pcoeff);           \
 616                         z_pcoeff++;                                             \
 617                         p += info->channels * PCM_##BIT##_BPS;                  \
 618                 }                                                               \
 619                 if (info->z_scale != Z_ONE)                                     \
 620                         v = Z_SCALE_##BIT(v, info->z_scale);                    \
 621                 else                                                            \
 622                         v >>= Z_COEFF_SHIFT - Z_GUARD_BIT_##BIT;                \
 623                 Z_CLIP_CHECK(v, BIT);                                           \
 624                 _PCM_WRITE_##SIGN##BIT##_##ENDIAN(dst, Z_CLAMP(v, BIT));        \
 625         } while (ch != 0);                                                      \
 626 }
 627
 628 #define Z_DECLARE(SIGN, BIT, ENDIAN)                                    \
 629         Z_DECLARE_LINEAR(SIGN, BIT, ENDIAN)                             \
 630         Z_DECLARE_SINC(SIGN, BIT, ENDIAN)                               \
 631         Z_DECLARE_SINC_POLYPHASE(SIGN, BIT, ENDIAN)
 632
 633 #if BYTE_ORDER == LITTLE_ENDIAN || defined(SND_FEEDER_MULTIFORMAT)
 634 Z_DECLARE(S, 16, LE)
 635 Z_DECLARE(S, 32, LE)
 636 #endif
 637 #if BYTE_ORDER == BIG_ENDIAN || defined(SND_FEEDER_MULTIFORMAT)
 638 Z_DECLARE(S, 16, BE)
 639 Z_DECLARE(S, 32, BE)
 640 #endif
 641 #ifdef SND_FEEDER_MULTIFORMAT
 642 Z_DECLARE(S,  8, NE)
 643 Z_DECLARE(S, 24, LE)
 644 Z_DECLARE(S, 24, BE)
 645 Z_DECLARE(U,  8, NE)
 646 Z_DECLARE(U, 16, LE)
 647 Z_DECLARE(U, 24, LE)
 648 Z_DECLARE(U, 32, LE)
 649 Z_DECLARE(U, 16, BE)
 650 Z_DECLARE(U, 24, BE)
 651 Z_DECLARE(U, 32, BE)
 652 #endif
 653
 654 enum {
 655         Z_RESAMPLER_ZOH,
 656         Z_RESAMPLER_LINEAR,
 657         Z_RESAMPLER_SINC,
 658         Z_RESAMPLER_SINC_POLYPHASE,
 659         Z_RESAMPLER_LAST
 660 };
 661
 662 #define Z_RESAMPLER_IDX(i)                                              \
 663         (Z_IS_SINC(i) ? Z_RESAMPLER_SINC : (i)->quality)
 664
 665 #define Z_RESAMPLER_ENTRY(SIGN, BIT, ENDIAN)                                    \
 666         {                                                                       \
 667             AFMT_##SIGN##BIT##_##ENDIAN,                                        \
 668             {                                                                   \
 669                 [Z_RESAMPLER_ZOH]    = z_feed_zoh,                              \
 670                 [Z_RESAMPLER_LINEAR] = z_feed_linear_##SIGN##BIT##ENDIAN,       \
 671                 [Z_RESAMPLER_SINC]   = z_feed_sinc_##SIGN##BIT##ENDIAN,         \
 672                 [Z_RESAMPLER_SINC_POLYPHASE]   =                                \
 673                     z_feed_sinc_polyphase_##SIGN##BIT##ENDIAN                   \
 674             }                                                                   \
 675         }
 676
 677 static const struct {
 678         uint32_t format;
 679         z_resampler_t resampler[Z_RESAMPLER_LAST];
 680 } z_resampler_tab[] = {
 681 #if BYTE_ORDER == LITTLE_ENDIAN || defined(SND_FEEDER_MULTIFORMAT)
 682         Z_RESAMPLER_ENTRY(S, 16, LE),
 683         Z_RESAMPLER_ENTRY(S, 32, LE),
 684 #endif
 685 #if BYTE_ORDER == BIG_ENDIAN || defined(SND_FEEDER_MULTIFORMAT)
 686         Z_RESAMPLER_ENTRY(S, 16, BE),
 687         Z_RESAMPLER_ENTRY(S, 32, BE),
 688 #endif
 689 #ifdef SND_FEEDER_MULTIFORMAT
 690         Z_RESAMPLER_ENTRY(S,  8, NE),
 691         Z_RESAMPLER_ENTRY(S, 24, LE),
 692         Z_RESAMPLER_ENTRY(S, 24, BE),
 693         Z_RESAMPLER_ENTRY(U,  8, NE),
 694         Z_RESAMPLER_ENTRY(U, 16, LE),
 695         Z_RESAMPLER_ENTRY(U, 24, LE),
 696         Z_RESAMPLER_ENTRY(U, 32, LE),
 697         Z_RESAMPLER_ENTRY(U, 16, BE),
 698         Z_RESAMPLER_ENTRY(U, 24, BE),
 699         Z_RESAMPLER_ENTRY(U, 32, BE),
 700 #endif
 701 };
 702
 703 #define Z_RESAMPLER_TAB_SIZE                                            \
 704         ((int32_t)(sizeof(z_resampler_tab) / sizeof(z_resampler_tab[0])))
 705
 706 static void
 707 z_resampler_reset(struct z_info *info)
 708 {
 709
 710         info->src = info->rsrc - (info->rsrc % ((feeder_rate_round > 0 &&
 711             info->rsrc > feeder_rate_round) ? feeder_rate_round : 1));
 712         info->dst = info->rdst - (info->rdst % ((feeder_rate_round > 0 &&
 713             info->rdst > feeder_rate_round) ? feeder_rate_round : 1));
 714         info->z_gx = 1;
 715         info->z_gy = 1;
 716         info->z_alpha = 0;
 717         info->z_resample = NULL;
 718         info->z_size = 1;
 719         info->z_coeff = NULL;
 720         info->z_dcoeff = NULL;
 721         if (info->z_pcoeff != NULL) {
 722                 free(info->z_pcoeff, M_DEVBUF);
 723                 info->z_pcoeff = NULL;
 724         }
 725         info->z_scale = Z_ONE;
 726         info->z_dx = Z_FULL_ONE;
 727         info->z_dy = Z_FULL_ONE;
 728 #ifdef Z_DIAGNOSTIC
 729         info->z_cycle = 0;
 730 #endif
 731         if (info->quality < Z_QUALITY_MIN)
 732                 info->quality = Z_QUALITY_MIN;
 733         else if (info->quality > Z_QUALITY_MAX)
 734                 info->quality = Z_QUALITY_MAX;
 735 }
 736
 737 #ifdef Z_PARANOID
 738 static int32_t
 739 z_resampler_sinc_len(struct z_info *info)
 740 {
 741         int32_t c, z, len, lmax;
 742
 743         if (!Z_IS_SINC(info))
 744                 return (1);
 745
 746         /*
 747          * A rather careful (or useless) way to calculate filter length.
 748          * Z_SINC_LEN() itself is accurate enough to do its job. Extra
 749          * sanity checking is not going to hurt though..
 750          */
 751         c = 0;
 752         z = info->z_dy;
 753         len = 0;
 754         lmax = z_coeff_tab[Z_SINC_COEFF_IDX(info)].len;
 755
 756         do {
 757                 c += z >> Z_SHIFT;
 758                 z &= Z_MASK;
 759                 z += info->z_dy;
 760         } while (c < lmax && ++len > 0);
 761
 762         if (len != Z_SINC_LEN(info)) {
 763 #ifdef _KERNEL
 764                 printf("%s(): sinc l=%d != Z_SINC_LEN=%d\n",
 765                     __func__, len, Z_SINC_LEN(info));
 766 #else
 767                 fprintf(stderr, "%s(): sinc l=%d != Z_SINC_LEN=%d\n",
 768                     __func__, len, Z_SINC_LEN(info));
 769                 return (-1);
 770 #endif
 771         }
 772
 773         return (len);
 774 }
 775 #else
 776 #define z_resampler_sinc_len(i)         (Z_IS_SINC(i) ? Z_SINC_LEN(i) : 1)
 777 #endif
 778
 779 #define Z_POLYPHASE_COEFF_SHIFT         0
 780
 781 /*
 782  * Pick suitable polynomial interpolators based on filter oversampled ratio
 783  * (2 ^ Z_DRIFT_SHIFT).
 784  */
 785 #if !(defined(Z_COEFF_INTERP_ZOH) || defined(Z_COEFF_INTERP_LINEAR) ||          \
 786     defined(Z_COEFF_INTERP_QUADRATIC) || defined(Z_COEFF_INTERP_HERMITE) ||     \
 787     defined(Z_COEFF_INTER_BSPLINE) || defined(Z_COEFF_INTERP_OPT32X) ||         \
 788     defined(Z_COEFF_INTERP_OPT16X) || defined(Z_COEFF_INTERP_OPT8X) ||          \
 789     defined(Z_COEFF_INTERP_OPT4X) || defined(Z_COEFF_INTERP_OPT2X))
 790 #if Z_DRIFT_SHIFT >= 6
 791 #define Z_COEFF_INTERP_BSPLINE          1
 792 #elif Z_DRIFT_SHIFT >= 5
 793 #define Z_COEFF_INTERP_OPT32X           1
 794 #elif Z_DRIFT_SHIFT == 4
 795 #define Z_COEFF_INTERP_OPT16X           1
 796 #elif Z_DRIFT_SHIFT == 3
 797 #define Z_COEFF_INTERP_OPT8X            1
 798 #elif Z_DRIFT_SHIFT == 2
 799 #define Z_COEFF_INTERP_OPT4X            1
 800 #elif Z_DRIFT_SHIFT == 1
 801 #define Z_COEFF_INTERP_OPT2X            1
 802 #else
 803 #error "Z_DRIFT_SHIFT screwed!"
 804 #endif
 805 #endif
 806
 807 /*
 808  * In classic polyphase mode, the actual coefficients for each phases need to
 809  * be calculated based on default prototype filters. For highly oversampled
 810  * filter, linear or quadradatic interpolator should be enough. Anything less
 811  * than that require 'special' interpolators to reduce interpolation errors.
 812  *
 813  * "Polynomial Interpolators for High-Quality Resampling of Oversampled Audio"
 814  *    by Olli Niemitalo
 815  *    - http://www.student.oulu.fi/~oniemita/dsp/deip.pdf
 816  *
 817  */
 818 static int32_t
 819 z_coeff_interpolate(int32_t z, int32_t *z_coeff)
 820 {
 821         int32_t coeff;
 822 #if defined(Z_COEFF_INTERP_ZOH)
 823
 824         /* 1-point, 0th-order (Zero Order Hold) */
 825         z = z;
 826         coeff = z_coeff[0];
 827 #elif defined(Z_COEFF_INTERP_LINEAR)
 828         int32_t zl0, zl1;
 829
 830         /* 2-point, 1st-order Linear */
 831         zl0 = z_coeff[0];
 832         zl1 = z_coeff[1] - z_coeff[0];
 833
 834         coeff = Z_RSHIFT((int64_t)zl1 * z, Z_SHIFT) + zl0;
 835 #elif defined(Z_COEFF_INTERP_QUADRATIC)
 836         int32_t zq0, zq1, zq2;
 837
 838         /* 3-point, 2nd-order Quadratic */
 839         zq0 = z_coeff[0];
 840         zq1 = z_coeff[1] - z_coeff[-1];
 841         zq2 = z_coeff[1] + z_coeff[-1] - (z_coeff[0] << 1);
 842
 843         coeff = Z_RSHIFT((Z_RSHIFT((int64_t)zq2 * z, Z_SHIFT) +
 844             zq1) * z, Z_SHIFT + 1) + zq0;
 845 #elif defined(Z_COEFF_INTERP_HERMITE)
 846         int32_t zh0, zh1, zh2, zh3;
 847
 848         /* 4-point, 3rd-order Hermite */
 849         zh0 = z_coeff[0];
 850         zh1 = z_coeff[1] - z_coeff[-1];
 851         zh2 = (z_coeff[-1] << 1) - (z_coeff[0] * 5) + (z_coeff[1] << 2) -
 852             z_coeff[2];
 853         zh3 = z_coeff[2] - z_coeff[-1] + ((z_coeff[0] - z_coeff[1]) * 3);
 854
 855         coeff = Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((int64_t)zh3 * z, Z_SHIFT) +
 856             zh2) * z, Z_SHIFT) + zh1) * z, Z_SHIFT + 1) + zh0;
 857 #elif defined(Z_COEFF_INTERP_BSPLINE)
 858         int32_t zb0, zb1, zb2, zb3;
 859
 860         /* 4-point, 3rd-order B-Spline */
 861         zb0 = Z_RSHIFT(0x15555555LL * (((int64_t)z_coeff[0] << 2) +
 862             z_coeff[-1] + z_coeff[1]), 30);
 863         zb1 = z_coeff[1] - z_coeff[-1];
 864         zb2 = z_coeff[-1] + z_coeff[1] - (z_coeff[0] << 1);
 865         zb3 = Z_RSHIFT(0x15555555LL * (((z_coeff[0] - z_coeff[1]) * 3) +
 866             z_coeff[2] - z_coeff[-1]), 30);
 867
 868         coeff = (Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((int64_t)zb3 * z, Z_SHIFT) +
 869             zb2) * z, Z_SHIFT) + zb1) * z, Z_SHIFT) + zb0 + 1) >> 1;
 870 #elif defined(Z_COEFF_INTERP_OPT32X)
 871         int32_t zoz, zoe1, zoe2, zoe3, zoo1, zoo2, zoo3;
 872         int32_t zoc0, zoc1, zoc2, zoc3, zoc4, zoc5;
 873
 874         /* 6-point, 5th-order Optimal 32x */
 875         zoz = z - (Z_ONE >> 1);
 876         zoe1 = z_coeff[1] + z_coeff[0];
 877         zoe2 = z_coeff[2] + z_coeff[-1];
 878         zoe3 = z_coeff[3] + z_coeff[-2];
 879         zoo1 = z_coeff[1] - z_coeff[0];
 880         zoo2 = z_coeff[2] - z_coeff[-1];
 881         zoo3 = z_coeff[3] - z_coeff[-2];
 882
 883         zoc0 = Z_RSHIFT((0x1ac2260dLL * zoe1) + (0x0526cdcaLL * zoe2) +
 884             (0x00170c29LL * zoe3), 30);
 885         zoc1 = Z_RSHIFT((0x14f8a49aLL * zoo1) + (0x0d6d1109LL * zoo2) +
 886             (0x008cd4dcLL * zoo3), 30);
 887         zoc2 = Z_RSHIFT((-0x0d3e94a4LL * zoe1) + (0x0bddded4LL * zoe2) +
 888             (0x0160b5d0LL * zoe3), 30);
 889         zoc3 = Z_RSHIFT((-0x0de10cc4LL * zoo1) + (0x019b2a7dLL * zoo2) +
 890             (0x01cfe914LL * zoo3), 30);
 891         zoc4 = Z_RSHIFT((0x02aa12d7LL * zoe1) + (-0x03ff1bb3LL * zoe2) +
 892             (0x015508ddLL * zoe3), 30);
 893         zoc5 = Z_RSHIFT((0x051d29e5LL * zoo1) + (-0x028e7647LL * zoo2) +
 894             (0x0082d81aLL * zoo3), 30);
 895
 896         coeff = Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT(
 897             (int64_t)zoc5 * zoz, Z_SHIFT) +
 898             zoc4) * zoz, Z_SHIFT) + zoc3) * zoz, Z_SHIFT) +
 899             zoc2) * zoz, Z_SHIFT) + zoc1) * zoz, Z_SHIFT) + zoc0;
 900 #elif defined(Z_COEFF_INTERP_OPT16X)
 901         int32_t zoz, zoe1, zoe2, zoe3, zoo1, zoo2, zoo3;
 902         int32_t zoc0, zoc1, zoc2, zoc3, zoc4, zoc5;
 903
 904         /* 6-point, 5th-order Optimal 16x */
 905         zoz = z - (Z_ONE >> 1);
 906         zoe1 = z_coeff[1] + z_coeff[0];
 907         zoe2 = z_coeff[2] + z_coeff[-1];
 908         zoe3 = z_coeff[3] + z_coeff[-2];
 909         zoo1 = z_coeff[1] - z_coeff[0];
 910         zoo2 = z_coeff[2] - z_coeff[-1];
 911         zoo3 = z_coeff[3] - z_coeff[-2];
 912
 913         zoc0 = Z_RSHIFT((0x1ac2260dLL * zoe1) + (0x0526cdcaLL * zoe2) +
 914             (0x00170c29LL * zoe3), 30);
 915         zoc1 = Z_RSHIFT((0x14f8a49aLL * zoo1) + (0x0d6d1109LL * zoo2) +
 916             (0x008cd4dcLL * zoo3), 30);
 917         zoc2 = Z_RSHIFT((-0x0d3e94a4LL * zoe1) + (0x0bddded4LL * zoe2) +
 918             (0x0160b5d0LL * zoe3), 30);
 919         zoc3 = Z_RSHIFT((-0x0de10cc4LL * zoo1) + (0x019b2a7dLL * zoo2) +
 920             (0x01cfe914LL * zoo3), 30);
 921         zoc4 = Z_RSHIFT((0x02aa12d7LL * zoe1) + (-0x03ff1bb3LL * zoe2) +
 922             (0x015508ddLL * zoe3), 30);
 923         zoc5 = Z_RSHIFT((0x051d29e5LL * zoo1) + (-0x028e7647LL * zoo2) +
 924             (0x0082d81aLL * zoo3), 30);
 925
 926         coeff = Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT(
 927             (int64_t)zoc5 * zoz, Z_SHIFT) +
 928             zoc4) * zoz, Z_SHIFT) + zoc3) * zoz, Z_SHIFT) +
 929             zoc2) * zoz, Z_SHIFT) + zoc1) * zoz, Z_SHIFT) + zoc0;
 930 #elif defined(Z_COEFF_INTERP_OPT8X)
 931         int32_t zoz, zoe1, zoe2, zoe3, zoo1, zoo2, zoo3;
 932         int32_t zoc0, zoc1, zoc2, zoc3, zoc4, zoc5;
 933
 934         /* 6-point, 5th-order Optimal 8x */
 935         zoz = z - (Z_ONE >> 1);
 936         zoe1 = z_coeff[1] + z_coeff[0];
 937         zoe2 = z_coeff[2] + z_coeff[-1];
 938         zoe3 = z_coeff[3] + z_coeff[-2];
 939         zoo1 = z_coeff[1] - z_coeff[0];
 940         zoo2 = z_coeff[2] - z_coeff[-1];
 941         zoo3 = z_coeff[3] - z_coeff[-2];
 942
 943         zoc0 = Z_RSHIFT((0x1aa9b47dLL * zoe1) + (0x053d9944LL * zoe2) +
 944             (0x0018b23fLL * zoe3), 30);
 945         zoc1 = Z_RSHIFT((0x14a104d1LL * zoo1) + (0x0d7d2504LL * zoo2) +
 946             (0x0094b599LL * zoo3), 30);
 947         zoc2 = Z_RSHIFT((-0x0d22530bLL * zoe1) + (0x0bb37a2cLL * zoe2) +
 948             (0x016ed8e0LL * zoe3), 30);
 949         zoc3 = Z_RSHIFT((-0x0d744b1cLL * zoo1) + (0x01649591LL * zoo2) +
 950             (0x01dae93aLL * zoo3), 30);
 951         zoc4 = Z_RSHIFT((0x02a7ee1bLL * zoe1) + (-0x03fbdb24LL * zoe2) +
 952             (0x0153ed07LL * zoe3), 30);
 953         zoc5 = Z_RSHIFT((0x04cf9b6cLL * zoo1) + (-0x0266b378LL * zoo2) +
 954             (0x007a7c26LL * zoo3), 30);
 955
 956         coeff = Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT(
 957             (int64_t)zoc5 * zoz, Z_SHIFT) +
 958             zoc4) * zoz, Z_SHIFT) + zoc3) * zoz, Z_SHIFT) +
 959             zoc2) * zoz, Z_SHIFT) + zoc1) * zoz, Z_SHIFT) + zoc0;
 960 #elif defined(Z_COEFF_INTERP_OPT4X)
 961         int32_t zoz, zoe1, zoe2, zoe3, zoo1, zoo2, zoo3;
 962         int32_t zoc0, zoc1, zoc2, zoc3, zoc4, zoc5;
 963
 964         /* 6-point, 5th-order Optimal 4x */
 965         zoz = z - (Z_ONE >> 1);
 966         zoe1 = z_coeff[1] + z_coeff[0];
 967         zoe2 = z_coeff[2] + z_coeff[-1];
 968         zoe3 = z_coeff[3] + z_coeff[-2];
 969         zoo1 = z_coeff[1] - z_coeff[0];
 970         zoo2 = z_coeff[2] - z_coeff[-1];
 971         zoo3 = z_coeff[3] - z_coeff[-2];
 972
 973         zoc0 = Z_RSHIFT((0x1a8eda43LL * zoe1) + (0x0556ee38LL * zoe2) +
 974             (0x001a3784LL * zoe3), 30);
 975         zoc1 = Z_RSHIFT((0x143d863eLL * zoo1) + (0x0d910e36LL * zoo2) +
 976             (0x009ca889LL * zoo3), 30);
 977         zoc2 = Z_RSHIFT((-0x0d026821LL * zoe1) + (0x0b837773LL * zoe2) +
 978             (0x017ef0c6LL * zoe3), 30);
 979         zoc3 = Z_RSHIFT((-0x0cef1502LL * zoo1) + (0x01207a8eLL * zoo2) +
 980             (0x01e936dbLL * zoo3), 30);
 981         zoc4 = Z_RSHIFT((0x029fe643LL * zoe1) + (-0x03ef3fc8LL * zoe2) +
 982             (0x014f5923LL * zoe3), 30);
 983         zoc5 = Z_RSHIFT((0x043a9d08LL * zoo1) + (-0x02154febLL * zoo2) +
 984             (0x00670dbdLL * zoo3), 30);
 985
 986         coeff = Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT(
 987             (int64_t)zoc5 * zoz, Z_SHIFT) +
 988             zoc4) * zoz, Z_SHIFT) + zoc3) * zoz, Z_SHIFT) +
 989             zoc2) * zoz, Z_SHIFT) + zoc1) * zoz, Z_SHIFT) + zoc0;
 990 #elif defined(Z_COEFF_INTERP_OPT2X)
 991         int32_t zoz, zoe1, zoe2, zoe3, zoo1, zoo2, zoo3;
 992         int32_t zoc0, zoc1, zoc2, zoc3, zoc4, zoc5;
 993
 994         /* 6-point, 5th-order Optimal 2x */
 995         zoz = z - (Z_ONE >> 1);
 996         zoe1 = z_coeff[1] + z_coeff[0];
 997         zoe2 = z_coeff[2] + z_coeff[-1];
 998         zoe3 = z_coeff[3] + z_coeff[-2];
 999         zoo1 = z_coeff[1] - z_coeff[0];
1000         zoo2 = z_coeff[2] - z_coeff[-1];
1001         zoo3 = z_coeff[3] - z_coeff[-2];
1002
1003         zoc0 = Z_RSHIFT((0x19edb6fdLL * zoe1) + (0x05ebd062LL * zoe2) +
1004             (0x00267881LL * zoe3), 30);
1005         zoc1 = Z_RSHIFT((0x1223af76LL * zoo1) + (0x0de3dd6bLL * zoo2) +
1006             (0x00d683cdLL * zoo3), 30);
1007         zoc2 = Z_RSHIFT((-0x0c3ee068LL * zoe1) + (0x0a5c3769LL * zoe2) +
1008             (0x01e2aceaLL * zoe3), 30);
1009         zoc3 = Z_RSHIFT((-0x0a8ab614LL * zoo1) + (-0x0019522eLL * zoo2) +
1010             (0x022cefc7LL * zoo3), 30);
1011         zoc4 = Z_RSHIFT((0x0276187dLL * zoe1) + (-0x03a801e8LL * zoe2) +
1012             (0x0131d935LL * zoe3), 30);
1013         zoc5 = Z_RSHIFT((0x02c373f5LL * zoo1) + (-0x01275f83LL * zoo2) +
1014             (0x0018ee79LL * zoo3), 30);
1015
1016         coeff = Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT(
1017             (int64_t)zoc5 * zoz, Z_SHIFT) +
1018             zoc4) * zoz, Z_SHIFT) + zoc3) * zoz, Z_SHIFT) +
1019             zoc2) * zoz, Z_SHIFT) + zoc1) * zoz, Z_SHIFT) + zoc0;
1020 #else
1021 #error "Interpolation type screwed!"
1022 #endif
1023
1024 #if Z_POLYPHASE_COEFF_SHIFT > 0
1025         coeff = Z_RSHIFT(coeff, Z_POLYPHASE_COEFF_SHIFT);
1026 #endif
1027         return (coeff);
1028 }
1029
1030 static int
1031 z_resampler_build_polyphase(struct z_info *info)
1032 {
1033         int32_t alpha, c, i, z, idx;
1034
1035         /* Let this be here first. */
1036         if (info->z_pcoeff != NULL) {
1037                 free(info->z_pcoeff, M_DEVBUF);
1038                 info->z_pcoeff = NULL;
1039         }
1040
1041         if (feeder_rate_polyphase_max < 1)
1042                 return (ENOTSUP);
1043
1044         if (((int64_t)info->z_size * info->z_gy * 2) >
1045             feeder_rate_polyphase_max) {
1046 #ifndef _KERNEL
1047                 fprintf(stderr, "Polyphase entries exceed: [%d/%d] %jd > %d\n",
1048                     info->z_gx, info->z_gy,
1049                     (intmax_t)info->z_size * info->z_gy * 2,
1050                     feeder_rate_polyphase_max);
1051 #endif
1052                 return (E2BIG);
1053         }
1054
1055         info->z_pcoeff = malloc(sizeof(int32_t) *
1056             info->z_size * info->z_gy * 2, M_DEVBUF, M_NOWAIT | M_ZERO);
1057         if (info->z_pcoeff == NULL)
1058                 return (ENOMEM);
1059
1060         for (alpha = 0; alpha < info->z_gy; alpha++) {
1061                 z = alpha * info->z_dx;
1062                 c = 0;
1063                 for (i = info->z_size; i != 0; i--) {
1064                         c += z >> Z_SHIFT;
1065                         z &= Z_MASK;
1066                         idx = (alpha * info->z_size * 2) +
1067                             (info->z_size * 2) - i;
1068                         info->z_pcoeff[idx] =
1069                             z_coeff_interpolate(z, info->z_coeff + c);
1070                         z += info->z_dy;
1071                 }
1072                 z = info->z_dy - (alpha * info->z_dx);
1073                 c = 0;
1074                 for (i = info->z_size; i != 0; i--) {
1075                         c += z >> Z_SHIFT;
1076                         z &= Z_MASK;
1077                         idx = (alpha * info->z_size * 2) + i - 1;
1078                         info->z_pcoeff[idx] =
1079                             z_coeff_interpolate(z, info->z_coeff + c);
1080                         z += info->z_dy;
1081                 }
1082         }
1083
1084 #ifndef _KERNEL
1085         fprintf(stderr, "Polyphase: [%d/%d] %d entries\n",
1086             info->z_gx, info->z_gy, info->z_size * info->z_gy * 2);
1087 #endif
1088
1089         return (0);
1090 }
1091
1092 static int
1093 z_resampler_setup(struct pcm_feeder *f)
1094 {
1095         struct z_info *info;
1096         int64_t gy2gx_max, gx2gy_max;
1097         uint32_t format;
1098         int32_t align, i, z_scale;
1099         int adaptive;
1100
1101         info = f->data;
1102         z_resampler_reset(info);
1103
1104         if (info->src == info->dst)
1105                 return (0);
1106
1107         /* Shrink by greatest common divisor. */
1108         i = z_gcd(info->src, info->dst);
1109         info->z_gx = info->src / i;
1110         info->z_gy = info->dst / i;
1111
1112         /* Too big, or too small. Bail out. */
1113         if (!(Z_FACTOR_SAFE(info->z_gx) && Z_FACTOR_SAFE(info->z_gy)))
1114                 return (EINVAL);
1115
1116         format = f->desc->in;
1117         adaptive = 0;
1118         z_scale = 0;
1119
1120         /*
1121          * Setup everything: filter length, conversion factor, etc.
1122          */
1123         if (Z_IS_SINC(info)) {
1124                 /*
1125                  * Downsampling, or upsampling scaling factor. As long as the
1126                  * factor can be represented by a fraction of 1 << Z_SHIFT,
1127                  * we're pretty much in business. Scaling is not needed for
1128                  * upsampling, so we just slap Z_ONE there.
1129                  */
1130                 if (info->z_gx > info->z_gy)
1131                         /*
1132                          * If the downsampling ratio is beyond sanity,
1133                          * enable semi-adaptive mode. Although handling
1134                          * extreme ratio is possible, the result of the
1135                          * conversion is just pointless, unworthy,
1136                          * nonsensical noises, etc.
1137                          */
1138                         if ((info->z_gx / info->z_gy) > Z_SINC_DOWNMAX)
1139                                 z_scale = Z_ONE / Z_SINC_DOWNMAX;
1140                         else
1141                                 z_scale = ((uint64_t)info->z_gy << Z_SHIFT) /
1142                                     info->z_gx;
1143                 else
1144                         z_scale = Z_ONE;
1145
1146                 /*
1147                  * This is actually impossible, unless anything above
1148                  * overflow.
1149                  */
1150                 if (z_scale < 1)
1151                         return (E2BIG);
1152
1153                 /*
1154                  * Calculate sample time/coefficients index drift. It is
1155                  * a constant for upsampling, but downsampling require
1156                  * heavy duty filtering with possible too long filters.
1157                  * If anything goes wrong, revisit again and enable
1158                  * adaptive mode.
1159                  */
1160 z_setup_adaptive_sinc:
1161                 if (info->z_pcoeff != NULL) {
1162                         free(info->z_pcoeff, M_DEVBUF);
1163                         info->z_pcoeff = NULL;
1164                 }
1165
1166                 if (adaptive == 0) {
1167                         info->z_dy = z_scale << Z_DRIFT_SHIFT;
1168                         if (info->z_dy < 1)
1169                                 return (E2BIG);
1170                         info->z_scale = z_scale;
1171                 } else {
1172                         info->z_dy = Z_FULL_ONE;
1173                         info->z_scale = Z_ONE;
1174                 }
1175
1176 #if 0
1177 #define Z_SCALE_DIV     10000
1178 #define Z_SCALE_LIMIT(s, v)                                             \
1179         ((((uint64_t)(s) * (v)) + (Z_SCALE_DIV >> 1)) / Z_SCALE_DIV)
1180
1181                 info->z_scale = Z_SCALE_LIMIT(info->z_scale, 9780);
1182 #endif
1183
1184                 /* Smallest drift increment. */
1185                 info->z_dx = info->z_dy / info->z_gy;
1186
1187                 /*
1188                  * Overflow or underflow. Try adaptive, let it continue and
1189                  * retry.
1190                  */
1191                 if (info->z_dx < 1) {
1192                         if (adaptive == 0) {
1193                                 adaptive = 1;
1194                                 goto z_setup_adaptive_sinc;
1195                         }
1196                         return (E2BIG);
1197                 }
1198
1199                 /*
1200                  * Round back output drift.
1201                  */
1202                 info->z_dy = info->z_dx * info->z_gy;
1203
1204                 for (i = 0; i < Z_COEFF_TAB_SIZE; i++) {
1205                         if (Z_SINC_COEFF_IDX(info) != i)
1206                                 continue;
1207                         /*
1208                          * Calculate required filter length and guard
1209                          * against possible abusive result. Note that
1210                          * this represents only 1/2 of the entire filter
1211                          * length.
1212                          */
1213                         info->z_size = z_resampler_sinc_len(info);
1214
1215                         /*
1216                          * Multiple of 2 rounding, for better accumulator
1217                          * performance.
1218                          */
1219                         info->z_size &= ~1;
1220
1221                         if (info->z_size < 2 || info->z_size > Z_SINC_MAX) {
1222                                 if (adaptive == 0) {
1223                                         adaptive = 1;
1224                                         goto z_setup_adaptive_sinc;
1225                                 }
1226                                 return (E2BIG);
1227                         }
1228                         info->z_coeff = z_coeff_tab[i].coeff + Z_COEFF_OFFSET;
1229                         info->z_dcoeff = z_coeff_tab[i].dcoeff;
1230                         break;
1231                 }
1232
1233                 if (info->z_coeff == NULL || info->z_dcoeff == NULL)
1234                         return (EINVAL);
1235         } else if (Z_IS_LINEAR(info)) {
1236                 /*
1237                  * Don't put much effort if we're doing linear interpolation.
1238                  * Just center the interpolation distance within Z_LINEAR_ONE,
1239                  * and be happy about it.
1240                  */
1241                 info->z_dx = Z_LINEAR_FULL_ONE / info->z_gy;
1242         }
1243
1244         /*
1245          * We're safe for now, lets continue.. Look for our resampler
1246          * depending on configured format and quality.
1247          */
1248         for (i = 0; i < Z_RESAMPLER_TAB_SIZE; i++) {
1249                 int ridx;
1250
1251                 if (AFMT_ENCODING(format) != z_resampler_tab[i].format)
1252                         continue;
1253                 if (Z_IS_SINC(info) && adaptive == 0 &&
1254                     z_resampler_build_polyphase(info) == 0)
1255                         ridx = Z_RESAMPLER_SINC_POLYPHASE;
1256                 else
1257                         ridx = Z_RESAMPLER_IDX(info);
1258                 info->z_resample = z_resampler_tab[i].resampler[ridx];
1259                 break;
1260         }
1261
1262         if (info->z_resample == NULL)
1263                 return (EINVAL);
1264
1265         info->bps = AFMT_BPS(format);
1266         align = info->channels * info->bps;
1267
1268         /*
1269          * Calculate largest value that can be fed into z_gy2gx() and
1270          * z_gx2gy() without causing (signed) 32bit overflow. z_gy2gx() will
1271          * be called early during feeding process to determine how much input
1272          * samples that is required to generate requested output, while
1273          * z_gx2gy() will be called just before samples filtering /
1274          * accumulation process based on available samples that has been
1275          * calculated using z_gx2gy().
1276          *
1277          * Now that is damn confusing, I guess ;-) .
1278          */
1279         gy2gx_max = (((uint64_t)info->z_gy * INT32_MAX) - info->z_gy + 1) /
1280             info->z_gx;
1281
1282         if ((gy2gx_max * align) > SND_FXDIV_MAX)
1283                 gy2gx_max = SND_FXDIV_MAX / align;
1284
1285         if (gy2gx_max < 1)
1286                 return (E2BIG);
1287
1288         gx2gy_max = (((uint64_t)info->z_gx * INT32_MAX) - info->z_gy) /
1289             info->z_gy;
1290
1291         if (gx2gy_max > INT32_MAX)
1292                 gx2gy_max = INT32_MAX;
1293
1294         if (gx2gy_max < 1)
1295                 return (E2BIG);
1296
1297         /*
1298          * Ensure that z_gy2gx() at its largest possible calculated value
1299          * (alpha = 0) will not cause overflow further late during z_gx2gy()
1300          * stage.
1301          */
1302         if (z_gy2gx(info, gy2gx_max) > _Z_GCAST(gx2gy_max))
1303                 return (E2BIG);
1304
1305         info->z_maxfeed = gy2gx_max * align;
1306
1307 #ifdef Z_USE_ALPHADRIFT
1308         info->z_startdrift = z_gy2gx(info, 1);
1309         info->z_alphadrift = z_drift(info, info->z_startdrift, 1);
1310 #endif
1311
1312         i = z_gy2gx(info, 1);
1313         info->z_full = z_roundpow2((info->z_size << 1) + i);
1314
1315         /*
1316          * Too big to be true, and overflowing left and right like mad ..
1317          */
1318         if ((info->z_full * align) < 1) {
1319                 if (adaptive == 0 && Z_IS_SINC(info)) {
1320                         adaptive = 1;
1321                         goto z_setup_adaptive_sinc;
1322                 }
1323                 return (E2BIG);
1324         }
1325
1326         /*
1327          * Increase full buffer size if its too small to reduce cyclic
1328          * buffer shifting in main conversion/feeder loop.
1329          */
1330         while (info->z_full < Z_RESERVOIR_MAX &&
1331             (info->z_full - (info->z_size << 1)) < Z_RESERVOIR)
1332                 info->z_full <<= 1;
1333
1334         /* Initialize buffer position. */
1335         info->z_mask = info->z_full - 1;
1336         info->z_start = z_prev(info, info->z_size << 1, 1);
1337         info->z_pos = z_next(info, info->z_start, 1);
1338
1339         /*
1340          * Allocate or reuse delay line buffer, whichever makes sense.
1341          */
1342         i = info->z_full * align;
1343         if (i < 1)
1344                 return (E2BIG);
1345
1346         if (info->z_delay == NULL || info->z_alloc < i ||
1347             i <= (info->z_alloc >> 1)) {
1348                 if (info->z_delay != NULL)
1349                         free(info->z_delay, M_DEVBUF);
1350                 info->z_delay = malloc(i, M_DEVBUF, M_NOWAIT | M_ZERO);
1351                 if (info->z_delay == NULL)
1352                         return (ENOMEM);
1353                 info->z_alloc = i;
1354         }
1355
1356         /*
1357          * Zero out head of buffer to avoid pops and clicks.
1358          */
1359         memset(info->z_delay, sndbuf_zerodata(f->desc->out),
1360             info->z_pos * align);
1361
1362 #ifdef Z_DIAGNOSTIC
1363         /*
1364          * XXX Debuging mess !@#$%^
1365          */
1366 #define dumpz(x)        fprintf(stderr, "\t%12s = %10u : %-11d\n",      \
1367                             "z_"__STRING(x), (uint32_t)info->z_##x,     \
1368                             (int32_t)info->z_##x)
1369         fprintf(stderr, "\n%s():\n", __func__);
1370         fprintf(stderr, "\tchannels=%d, bps=%d, format=0x%08x, quality=%d\n",
1371             info->channels, info->bps, format, info->quality);
1372         fprintf(stderr, "\t%d (%d) -> %d (%d), ",
1373             info->src, info->rsrc, info->dst, info->rdst);
1374         fprintf(stderr, "[%d/%d]\n", info->z_gx, info->z_gy);
1375         fprintf(stderr, "\tminreq=%d, ", z_gy2gx(info, 1));
1376         if (adaptive != 0)
1377                 z_scale = Z_ONE;
1378         fprintf(stderr, "factor=0x%08x/0x%08x (%f)\n",
1379             z_scale, Z_ONE, (double)z_scale / Z_ONE);
1380         fprintf(stderr, "\tbase_length=%d, ", Z_SINC_BASE_LEN(info));
1381         fprintf(stderr, "adaptive=%s\n", (adaptive != 0) ? "YES" : "NO");
1382         dumpz(size);
1383         dumpz(alloc);
1384         if (info->z_alloc < 1024)
1385                 fprintf(stderr, "\t%15s%10d Bytes\n",
1386                     "", info->z_alloc);
1387         else if (info->z_alloc < (1024 << 10))
1388                 fprintf(stderr, "\t%15s%10d KBytes\n",
1389                     "", info->z_alloc >> 10);
1390         else if (info->z_alloc < (1024 << 20))
1391                 fprintf(stderr, "\t%15s%10d MBytes\n",
1392                     "", info->z_alloc >> 20);
1393         else
1394                 fprintf(stderr, "\t%15s%10d GBytes\n",
1395                     "", info->z_alloc >> 30);
1396         fprintf(stderr, "\t%12s   %10d (min output samples)\n",
1397             "",
1398             (int32_t)z_gx2gy(info, info->z_full - (info->z_size << 1)));
1399         fprintf(stderr, "\t%12s   %10d (min allocated output samples)\n",
1400             "",
1401             (int32_t)z_gx2gy(info, (info->z_alloc / align) -
1402             (info->z_size << 1)));
1403         fprintf(stderr, "\t%12s = %10d\n",
1404             "z_gy2gx()", (int32_t)z_gy2gx(info, 1));
1405         fprintf(stderr, "\t%12s = %10d -> z_gy2gx() -> %d\n",
1406             "Max", (int32_t)gy2gx_max, (int32_t)z_gy2gx(info, gy2gx_max));
1407         fprintf(stderr, "\t%12s = %10d\n",
1408             "z_gx2gy()", (int32_t)z_gx2gy(info, 1));
1409         fprintf(stderr, "\t%12s = %10d -> z_gx2gy() -> %d\n",
1410             "Max", (int32_t)gx2gy_max, (int32_t)z_gx2gy(info, gx2gy_max));
1411         dumpz(maxfeed);
1412         dumpz(full);
1413         dumpz(start);
1414         dumpz(pos);
1415         dumpz(scale);
1416         fprintf(stderr, "\t%12s   %10f\n", "",
1417             (double)info->z_scale / Z_ONE);
1418         dumpz(dx);
1419         fprintf(stderr, "\t%12s   %10f\n", "",
1420             (double)info->z_dx / info->z_dy);
1421         dumpz(dy);
1422         fprintf(stderr, "\t%12s   %10d (drift step)\n", "",
1423             info->z_dy >> Z_SHIFT);
1424         fprintf(stderr, "\t%12s   %10d (scaling differences)\n", "",
1425             (z_scale << Z_DRIFT_SHIFT) - info->z_dy);
1426         fprintf(stderr, "\t%12s = %u bytes\n",
1427             "intpcm32_t", sizeof(intpcm32_t));
1428         fprintf(stderr, "\t%12s = 0x%08x, smallest=%.16lf\n",
1429             "Z_ONE", Z_ONE, (double)1.0 / (double)Z_ONE);
1430 #endif
1431
1432         return (0);
1433 }
1434
1435 static int
1436 z_resampler_set(struct pcm_feeder *f, int what, int32_t value)
1437 {
1438         struct z_info *info;
1439         int32_t oquality;
1440
1441         info = f->data;
1442
1443         switch (what) {
1444         case Z_RATE_SRC:
1445                 if (value < feeder_rate_min || value > feeder_rate_max)
1446                         return (E2BIG);
1447                 if (value == info->rsrc)
1448                         return (0);
1449                 info->rsrc = value;
1450                 break;
1451         case Z_RATE_DST:
1452                 if (value < feeder_rate_min || value > feeder_rate_max)
1453                         return (E2BIG);
1454                 if (value == info->rdst)
1455                         return (0);
1456                 info->rdst = value;
1457                 break;
1458         case Z_RATE_QUALITY:
1459                 if (value < Z_QUALITY_MIN || value > Z_QUALITY_MAX)
1460                         return (EINVAL);
1461                 if (value == info->quality)
1462                         return (0);
1463                 /*
1464                  * If we failed to set the requested quality, restore
1465                  * the old one. We cannot afford leaving it broken since
1466                  * passive feeder chains like vchans never reinitialize
1467                  * itself.
1468                  */
1469                 oquality = info->quality;
1470                 info->quality = value;
1471                 if (z_resampler_setup(f) == 0)
1472                         return (0);
1473                 info->quality = oquality;
1474                 break;
1475         case Z_RATE_CHANNELS:
1476                 if (value < SND_CHN_MIN || value > SND_CHN_MAX)
1477                         return (EINVAL);
1478                 if (value == info->channels)
1479                         return (0);
1480                 info->channels = value;
1481                 break;
1482         default:
1483                 return (EINVAL);
1484                 break;
1485         }
1486
1487         return (z_resampler_setup(f));
1488 }
1489
1490 static int
1491 z_resampler_get(struct pcm_feeder *f, int what)
1492 {
1493         struct z_info *info;
1494
1495         info = f->data;
1496
1497         switch (what) {
1498         case Z_RATE_SRC:
1499                 return (info->rsrc);
1500                 break;
1501         case Z_RATE_DST:
1502                 return (info->rdst);
1503                 break;
1504         case Z_RATE_QUALITY:
1505                 return (info->quality);
1506                 break;
1507         case Z_RATE_CHANNELS:
1508                 return (info->channels);
1509                 break;
1510         default:
1511                 break;
1512         }
1513
1514         return (-1);
1515 }
1516
1517 static int
1518 z_resampler_init(struct pcm_feeder *f)
1519 {
1520         struct z_info *info;
1521         int ret;
1522
1523         if (f->desc->in != f->desc->out)
1524                 return (EINVAL);
1525
1526         info = malloc(sizeof(*info), M_DEVBUF, M_NOWAIT | M_ZERO);
1527         if (info == NULL)
1528                 return (ENOMEM);
1529
1530         info->rsrc = Z_RATE_DEFAULT;
1531         info->rdst = Z_RATE_DEFAULT;
1532         info->quality = feeder_rate_quality;
1533         info->channels = AFMT_CHANNEL(f->desc->in);
1534
1535         f->data = info;
1536
1537         ret = z_resampler_setup(f);
1538         if (ret != 0) {
1539                 if (info->z_pcoeff != NULL)
1540                         free(info->z_pcoeff, M_DEVBUF);
1541                 if (info->z_delay != NULL)
1542                         free(info->z_delay, M_DEVBUF);
1543                 free(info, M_DEVBUF);
1544                 f->data = NULL;
1545         }
1546
1547         return (ret);
1548 }
1549
1550 static int
1551 z_resampler_free(struct pcm_feeder *f)
1552 {
1553         struct z_info *info;
1554
1555         info = f->data;
1556         if (info != NULL) {
1557                 if (info->z_pcoeff != NULL)
1558                         free(info->z_pcoeff, M_DEVBUF);
1559                 if (info->z_delay != NULL)
1560                         free(info->z_delay, M_DEVBUF);
1561                 free(info, M_DEVBUF);
1562         }
1563
1564         f->data = NULL;
1565
1566         return (0);
1567 }
1568
1569 static uint32_t
1570 z_resampler_feed_internal(struct pcm_feeder *f, struct pcm_channel *c,
1571     uint8_t *b, uint32_t count, void *source)
1572 {
1573         struct z_info *info;
1574         int32_t alphadrift, startdrift, reqout, ocount, reqin, align;
1575         int32_t fetch, fetched, start, cp;
1576         uint8_t *dst;
1577
1578         info = f->data;
1579         if (info->z_resample == NULL)
1580                 return (z_feed(f->source, c, b, count, source));
1581
1582         /*
1583          * Calculate sample size alignment and amount of sample output.
1584          * We will do everything in sample domain, but at the end we
1585          * will jump back to byte domain.
1586          */
1587         align = info->channels * info->bps;
1588         ocount = SND_FXDIV(count, align);
1589         if (ocount == 0)
1590                 return (0);
1591
1592         /*
1593          * Calculate amount of input samples that is needed to generate
1594          * exact amount of output.
1595          */
1596         reqin = z_gy2gx(info, ocount) - z_fetched(info);
1597
1598 #ifdef Z_USE_ALPHADRIFT
1599         startdrift = info->z_startdrift;
1600         alphadrift = info->z_alphadrift;
1601 #else
1602         startdrift = _Z_GY2GX(info, 0, 1);
1603         alphadrift = z_drift(info, startdrift, 1);
1604 #endif
1605
1606         dst = b;
1607
1608         do {
1609                 if (reqin != 0) {
1610                         fetch = z_min(z_free(info), reqin);
1611                         if (fetch == 0) {
1612                                 /*
1613                                  * No more free spaces, so wind enough
1614                                  * samples back to the head of delay line
1615                                  * in byte domain.
1616                                  */
1617                                 fetched = z_fetched(info);
1618                                 start = z_prev(info, info->z_start,
1619                                     (info->z_size << 1) - 1);
1620                                 cp = (info->z_size << 1) + fetched;
1621                                 z_copy(info->z_delay + (start * align),
1622                                     info->z_delay, cp * align);
1623                                 info->z_start =
1624                                     z_prev(info, info->z_size << 1, 1);
1625                                 info->z_pos =
1626                                     z_next(info, info->z_start, fetched + 1);
1627                                 fetch = z_min(z_free(info), reqin);
1628 #ifdef Z_DIAGNOSTIC
1629                                 if (1) {
1630                                         static uint32_t kk = 0;
1631                                         fprintf(stderr,
1632                                             "Buffer Move: "
1633                                             "start=%d fetched=%d cp=%d "
1634                                             "cycle=%u [%u]\r",
1635                                             start, fetched, cp, info->z_cycle,
1636                                             ++kk);
1637                                 }
1638                                 info->z_cycle = 0;
1639 #endif
1640                         }
1641                         if (fetch != 0) {
1642                                 /*
1643                                  * Fetch in byte domain and jump back
1644                                  * to sample domain.
1645                                  */
1646                                 fetched = SND_FXDIV(z_feed(f->source, c,
1647                                     info->z_delay + (info->z_pos * align),
1648                                     fetch * align, source), align);
1649                                 /*
1650                                  * Prepare to convert fetched buffer,
1651                                  * or mark us done if we cannot fulfill
1652                                  * the request.
1653                                  */
1654                                 reqin -= fetched;
1655                                 info->z_pos += fetched;
1656                                 if (fetched != fetch)
1657                                         reqin = 0;
1658                         }
1659                 }
1660
1661                 reqout = z_min(z_gx2gy(info, z_fetched(info)), ocount);
1662                 if (reqout != 0) {
1663                         ocount -= reqout;
1664
1665                         /*
1666                          * Drift.. drift.. drift..
1667                          *
1668                          * Notice that there are 2 methods of doing the drift
1669                          * operations: The former is much cleaner (in a sense
1670                          * of mathematical readings of my eyes), but slower
1671                          * due to integer division in z_gy2gx(). Nevertheless,
1672                          * both should give the same exact accurate drifting
1673                          * results, so the later is favourable.
1674                          */
1675                         do {
1676                                 info->z_resample(info, dst);
1677 #if 0
1678                                 startdrift = z_gy2gx(info, 1);
1679                                 alphadrift = z_drift(info, startdrift, 1);
1680                                 info->z_start += startdrift;
1681                                 info->z_alpha += alphadrift;
1682 #else
1683                                 info->z_alpha += alphadrift;
1684                                 if (info->z_alpha < info->z_gy)
1685                                         info->z_start += startdrift;
1686                                 else {
1687                                         info->z_start += startdrift - 1;
1688                                         info->z_alpha -= info->z_gy;
1689                                 }
1690 #endif
1691                                 dst += align;
1692 #ifdef Z_DIAGNOSTIC
1693                                 info->z_cycle++;
1694 #endif
1695                         } while (--reqout != 0);
1696                 }
1697         } while (reqin != 0 && ocount != 0);
1698
1699         /*
1700          * Back to byte domain..
1701          */
1702         return (dst - b);
1703 }
1704
1705 static int
1706 z_resampler_feed(struct pcm_feeder *f, struct pcm_channel *c, uint8_t *b,
1707     uint32_t count, void *source)
1708 {
1709         uint32_t feed, maxfeed, left;
1710
1711         /*
1712          * Split count to smaller chunks to avoid possible 32bit overflow.
1713          */
1714         maxfeed = ((struct z_info *)(f->data))->z_maxfeed;
1715         left = count;
1716
1717         do {
1718                 feed = z_resampler_feed_internal(f, c, b,
1719                     z_min(maxfeed, left), source);
1720                 b += feed;
1721                 left -= feed;
1722         } while (left != 0 && feed != 0);
1723
1724         return (count - left);
1725 }
1726
1727 static struct pcm_feederdesc feeder_rate_desc[] = {
1728         { FEEDER_RATE, 0, 0, 0, 0 },
1729         { 0, 0, 0, 0, 0 },
1730 };
1731
1732 static kobj_method_t feeder_rate_methods[] = {
1733         KOBJMETHOD(feeder_init,         z_resampler_init),
1734         KOBJMETHOD(feeder_free,         z_resampler_free),
1735         KOBJMETHOD(feeder_set,          z_resampler_set),
1736         KOBJMETHOD(feeder_get,          z_resampler_get),
1737         KOBJMETHOD(feeder_feed,         z_resampler_feed),
1738         KOBJMETHOD_END
1739 };
1740
1741 FEEDER_DECLARE(feeder_rate, NULL);