From 5028f22f6eb0579890689655285a4778b4ffc460 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Tue, 4 Nov 2014 11:35:15 +0200 Subject: [PATCH] Switch to CRC-32C in WAL and other places. The old algorithm was found to not be the usual CRC-32 algorithm, used by Ethernet et al. We were using a non-reflected lookup table with code meant for a reflected lookup table. That's a strange combination that AFAICS does not correspond to any bit-wise CRC calculation, which makes it difficult to reason about its properties. Although it has worked well in practice, seems safer to use a well-known algorithm. Since we're changing the algorithm anyway, we might as well choose a different polynomial. The Castagnoli polynomial has better error-correcting properties than the traditional CRC-32 polynomial, even if we had implemented it correctly. Another reason for picking that is that some new CPUs have hardware support for calculating CRC-32C, but not CRC-32, let alone our strange variant of it. This patch doesn't add any support for such hardware, but a future patch could now do that. The old algorithm is kept around for tsquery and pg_trgm, which use the values in indexes that need to remain compatible so that pg_upgrade works. While we're at it, share the old lookup table for CRC-32 calculation between hstore, ltree and core. They all use the same table, so might as well. --- contrib/hstore/Makefile | 2 +- contrib/hstore/crc32.c | 106 ------------------ contrib/hstore/crc32.h | 13 --- contrib/hstore/hstore_gist.c | 16 ++- contrib/ltree/crc32.c | 114 ++++---------------- contrib/pg_trgm/trgm_op.c | 6 +- src/backend/access/transam/twophase.c | 24 ++--- src/backend/access/transam/xlog.c | 50 ++++----- src/backend/access/transam/xlogreader.c | 12 +-- src/backend/replication/logical/snapbuild.c | 28 ++--- src/backend/replication/slot.c | 20 ++-- src/backend/utils/adt/tsgistidx.c | 6 +- src/backend/utils/adt/tsquery.c | 12 +-- src/backend/utils/cache/relmapper.c | 16 +-- src/bin/pg_controldata/pg_controldata.c | 12 +-- src/bin/pg_resetxlog/pg_resetxlog.c | 30 +++--- src/include/access/xlog_internal.h | 2 +- src/include/catalog/catversion.h | 2 +- src/include/utils/pg_crc.h | 93 +++++++++++++--- src/include/utils/pg_crc_tables.h | 77 ++++++++++++- 20 files changed, 300 insertions(+), 341 deletions(-) delete mode 100644 contrib/hstore/crc32.c delete mode 100644 contrib/hstore/crc32.h diff --git a/contrib/hstore/Makefile b/contrib/hstore/Makefile index 3193668a0e7..82908de9da6 100644 --- a/contrib/hstore/Makefile +++ b/contrib/hstore/Makefile @@ -2,7 +2,7 @@ MODULE_big = hstore OBJS = hstore_io.o hstore_op.o hstore_gist.o hstore_gin.o hstore_compat.o \ - crc32.o $(WIN32RES) + $(WIN32RES) EXTENSION = hstore DATA = hstore--1.3.sql hstore--1.2--1.3.sql \ diff --git a/contrib/hstore/crc32.c b/contrib/hstore/crc32.c deleted file mode 100644 index c82fc664723..00000000000 --- a/contrib/hstore/crc32.c +++ /dev/null @@ -1,106 +0,0 @@ -/* - * contrib/hstore/crc32.c - * - * Both POSIX and CRC32 checksums */ - -#include -#include -#include - -#include "crc32.h" - -/* - * This code implements the AUTODIN II polynomial - * The variable corresponding to the macro argument "crc" should - * be an unsigned long. - * Original code by Spencer Garrett - */ - -#define _CRC32_(crc, ch) (crc = (crc >> 8) ^ crc32tab[(crc ^ (ch)) & 0xff]) - -/* generated using the AUTODIN II polynomial - * x^32 + x^26 + x^23 + x^22 + x^16 + - * x^12 + x^11 + x^10 + x^8 + x^7 + x^5 + x^4 + x^2 + x^1 + 1 - */ - -static const unsigned int crc32tab[256] = { - 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, - 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, - 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, - 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, - 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, - 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, - 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, - 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, - 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, - 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, - 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, - 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, - 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, - 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, - 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, - 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, - 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, - 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, - 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, - 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, - 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, - 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, - 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, - 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, - 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, - 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, - 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, - 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, - 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, - 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, - 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, - 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, - 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, - 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, - 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, - 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, - 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, - 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, - 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, - 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, - 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, - 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, - 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, - 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, - 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, - 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, - 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, - 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, - 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, - 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, - 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, - 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, - 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, - 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, - 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, - 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, - 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, - 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, - 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, - 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, - 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, - 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, - 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, - 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d, -}; - -unsigned int -crc32_sz(char *buf, int size) -{ - unsigned int crc = ~((unsigned int) 0); - char *p; - int len, - nr; - - len = 0; - nr = size; - for (len += nr, p = buf; nr--; ++p) - _CRC32_(crc, *p); - return ~crc; -} diff --git a/contrib/hstore/crc32.h b/contrib/hstore/crc32.h deleted file mode 100644 index f5bfd82517f..00000000000 --- a/contrib/hstore/crc32.h +++ /dev/null @@ -1,13 +0,0 @@ -/* - * contrib/hstore/crc32.h - */ -#ifndef _CRC32_H -#define _CRC32_H - -/* Returns crc32 of data block */ -extern unsigned int crc32_sz(char *buf, int size); - -/* Returns crc32 of null-terminated string */ -#define crc32(buf) crc32_sz((buf),strlen(buf)) - -#endif diff --git a/contrib/hstore/hstore_gist.c b/contrib/hstore/hstore_gist.c index d4a9aaa4c13..876b4359c6e 100644 --- a/contrib/hstore/hstore_gist.c +++ b/contrib/hstore/hstore_gist.c @@ -6,8 +6,8 @@ #include "access/gist.h" #include "access/skey.h" #include "catalog/pg_type.h" +#include "utils/pg_crc.h" -#include "crc32.h" #include "hstore.h" /* bigint defines */ @@ -68,6 +68,20 @@ typedef struct #define WISH_F(a,b,c) (double)( -(double)(((a)-(b))*((a)-(b))*((a)-(b)))*(c) ) +/* shorthand for calculating CRC-32 of a single chunk of data. */ +static pg_crc32 +crc32_sz(char *buf, int size) +{ + pg_crc32 crc; + + INIT_TRADITIONAL_CRC32(crc); + COMP_TRADITIONAL_CRC32(crc, buf, size); + FIN_TRADITIONAL_CRC32(crc); + + return crc; +} + + PG_FUNCTION_INFO_V1(ghstore_in); PG_FUNCTION_INFO_V1(ghstore_out); diff --git a/contrib/ltree/crc32.c b/contrib/ltree/crc32.c index ea1a661fefb..1c08d264f72 100644 --- a/contrib/ltree/crc32.c +++ b/contrib/ltree/crc32.c @@ -1,7 +1,12 @@ -/* Both POSIX and CRC32 checksums */ - /* contrib/ltree/crc32.c */ +/* + * Implements CRC-32, as used in ltree. + * + * Note that the CRC is used in the on-disk format of GiST indexes, so we + * must stay backwards-compatible! + */ + #include "postgres.h" #include @@ -15,100 +20,23 @@ #define TOLOWER(x) (x) #endif +#include "utils/pg_crc.h" #include "crc32.h" -/* - * This code implements the AUTODIN II polynomial - * The variable corresponding to the macro argument "crc" should - * be an unsigned long. - * Oroginal code by Spencer Garrett - */ - -#define _CRC32_(crc, ch) ((crc) = ((crc) >> 8) ^ crc32tab[((crc) ^ (ch)) & 0xff]) - -/* generated using the AUTODIN II polynomial - * x^32 + x^26 + x^23 + x^22 + x^16 + - * x^12 + x^11 + x^10 + x^8 + x^7 + x^5 + x^4 + x^2 + x^1 + 1 - */ - -static const unsigned int crc32tab[256] = { - 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, - 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, - 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, - 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, - 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, - 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, - 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, - 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, - 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, - 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, - 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, - 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, - 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, - 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, - 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, - 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, - 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, - 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, - 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, - 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, - 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, - 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, - 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, - 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, - 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, - 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, - 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, - 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, - 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, - 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, - 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, - 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, - 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, - 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, - 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, - 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, - 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, - 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, - 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, - 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, - 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, - 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, - 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, - 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, - 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, - 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, - 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, - 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, - 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, - 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, - 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, - 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, - 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, - 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, - 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, - 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, - 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, - 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, - 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, - 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, - 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, - 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, - 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, - 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d, -}; - unsigned int ltree_crc32_sz(char *buf, int size) { - unsigned int crc = ~((unsigned int) 0); - char *p; - int len, - nr; - - len = 0; - nr = size; - for (len += nr, p = buf; nr--; ++p) - _CRC32_(crc, TOLOWER((unsigned int) *p)); - return ~crc; + pg_crc32 crc; + char *p = buf; + + INIT_TRADITIONAL_CRC32(crc); + while (size > 0) + { + char c = (char) TOLOWER(*p); + COMP_TRADITIONAL_CRC32(crc, &c, 1); + size--; + p++; + } + FIN_TRADITIONAL_CRC32(crc); + return (unsigned int) crc; } diff --git a/contrib/pg_trgm/trgm_op.c b/contrib/pg_trgm/trgm_op.c index df15b52b4a1..5ec7f265240 100644 --- a/contrib/pg_trgm/trgm_op.c +++ b/contrib/pg_trgm/trgm_op.c @@ -108,9 +108,9 @@ compact_trigram(trgm *tptr, char *str, int bytelen) { pg_crc32 crc; - INIT_CRC32(crc); - COMP_CRC32(crc, str, bytelen); - FIN_CRC32(crc); + INIT_LEGACY_CRC32(crc); + COMP_LEGACY_CRC32(crc, str, bytelen); + FIN_LEGACY_CRC32(crc); /* * use only 3 upper bytes from crc, hope, it's good enough hashing diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index d5409a64210..c4069c39a20 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -847,9 +847,9 @@ TwoPhaseGetDummyProc(TransactionId xid) * 6. TwoPhaseRecordOnDisk * 7. ... * 8. TwoPhaseRecordOnDisk (end sentinel, rmid == TWOPHASE_RM_END_ID) - * 9. CRC32 + * 9. checksum (CRC-32C) * - * Each segment except the final CRC32 is MAXALIGN'd. + * Each segment except the final checksum is MAXALIGN'd. */ /* @@ -1056,11 +1056,11 @@ EndPrepare(GlobalTransaction gxact) path))); /* Write data to file, and calculate CRC as we pass over it */ - INIT_CRC32(statefile_crc); + INIT_CRC32C(statefile_crc); for (record = records.head; record != NULL; record = record->next) { - COMP_CRC32(statefile_crc, record->data, record->len); + COMP_CRC32C(statefile_crc, record->data, record->len); if ((write(fd, record->data, record->len)) != record->len) { CloseTransientFile(fd); @@ -1070,7 +1070,7 @@ EndPrepare(GlobalTransaction gxact) } } - FIN_CRC32(statefile_crc); + FIN_CRC32C(statefile_crc); /* * Write a deliberately bogus CRC to the state file; this is just paranoia @@ -1289,13 +1289,13 @@ ReadTwoPhaseFile(TransactionId xid, bool give_warnings) return NULL; } - INIT_CRC32(calc_crc); - COMP_CRC32(calc_crc, buf, crc_offset); - FIN_CRC32(calc_crc); + INIT_CRC32C(calc_crc); + COMP_CRC32C(calc_crc, buf, crc_offset); + FIN_CRC32C(calc_crc); file_crc = *((pg_crc32 *) (buf + crc_offset)); - if (!EQ_CRC32(calc_crc, file_crc)) + if (!EQ_CRC32C(calc_crc, file_crc)) { pfree(buf); return NULL; @@ -1540,9 +1540,9 @@ RecreateTwoPhaseFile(TransactionId xid, void *content, int len) int fd; /* Recompute CRC */ - INIT_CRC32(statefile_crc); - COMP_CRC32(statefile_crc, content, len); - FIN_CRC32(statefile_crc); + INIT_CRC32C(statefile_crc); + COMP_CRC32C(statefile_crc, content, len); + FIN_CRC32C(statefile_crc); TwoPhaseFilePath(path, xid); diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 3c9aeaec1e7..3160db72458 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1059,9 +1059,9 @@ begin:; * the whole record in the order: rdata, then backup blocks, then record * header. */ - INIT_CRC32(rdata_crc); + INIT_CRC32C(rdata_crc); for (rdt = rdata; rdt != NULL; rdt = rdt->next) - COMP_CRC32(rdata_crc, rdt->data, rdt->len); + COMP_CRC32C(rdata_crc, rdt->data, rdt->len); /* * Construct record header (prev-link is filled in later, after reserving @@ -1076,7 +1076,7 @@ begin:; rechdr->xl_info = info; rechdr->xl_rmid = rmid; rechdr->xl_prev = InvalidXLogRecPtr; - COMP_CRC32(rdata_crc, ((char *) rechdr), offsetof(XLogRecord, xl_prev)); + COMP_CRC32C(rdata_crc, ((char *) rechdr), offsetof(XLogRecord, xl_prev)); hdr_rdt.next = rdata; hdr_rdt.data = (char *) rechdr; @@ -1193,8 +1193,8 @@ begin:; * Now that xl_prev has been filled in, finish CRC calculation of the * record header. */ - COMP_CRC32(rdata_crc, ((char *) &rechdr->xl_prev), sizeof(XLogRecPtr)); - FIN_CRC32(rdata_crc); + COMP_CRC32C(rdata_crc, ((char *) &rechdr->xl_prev), sizeof(XLogRecPtr)); + FIN_CRC32C(rdata_crc); rechdr->xl_crc = rdata_crc; /* @@ -4344,11 +4344,11 @@ WriteControlFile(void) ControlFile->float8ByVal = FLOAT8PASSBYVAL; /* Contents are protected with a CRC */ - INIT_CRC32(ControlFile->crc); - COMP_CRC32(ControlFile->crc, - (char *) ControlFile, - offsetof(ControlFileData, crc)); - FIN_CRC32(ControlFile->crc); + INIT_CRC32C(ControlFile->crc); + COMP_CRC32C(ControlFile->crc, + (char *) ControlFile, + offsetof(ControlFileData, crc)); + FIN_CRC32C(ControlFile->crc); /* * We write out PG_CONTROL_SIZE bytes into pg_control, zero-padding the @@ -4444,13 +4444,13 @@ ReadControlFile(void) errhint("It looks like you need to initdb."))); /* Now check the CRC. */ - INIT_CRC32(crc); - COMP_CRC32(crc, - (char *) ControlFile, - offsetof(ControlFileData, crc)); - FIN_CRC32(crc); + INIT_CRC32C(crc); + COMP_CRC32C(crc, + (char *) ControlFile, + offsetof(ControlFileData, crc)); + FIN_CRC32C(crc); - if (!EQ_CRC32(crc, ControlFile->crc)) + if (!EQ_CRC32C(crc, ControlFile->crc)) ereport(FATAL, (errmsg("incorrect checksum in control file"))); @@ -4593,11 +4593,11 @@ UpdateControlFile(void) { int fd; - INIT_CRC32(ControlFile->crc); - COMP_CRC32(ControlFile->crc, - (char *) ControlFile, - offsetof(ControlFileData, crc)); - FIN_CRC32(ControlFile->crc); + INIT_CRC32C(ControlFile->crc); + COMP_CRC32C(ControlFile->crc, + (char *) ControlFile, + offsetof(ControlFileData, crc)); + FIN_CRC32C(ControlFile->crc); fd = BasicOpenFile(XLOG_CONTROL_FILE, O_RDWR | PG_BINARY, @@ -4975,10 +4975,10 @@ BootStrapXLOG(void) record->xl_rmid = RM_XLOG_ID; memcpy(XLogRecGetData(record), &checkPoint, sizeof(checkPoint)); - INIT_CRC32(crc); - COMP_CRC32(crc, &checkPoint, sizeof(checkPoint)); - COMP_CRC32(crc, (char *) record, offsetof(XLogRecord, xl_crc)); - FIN_CRC32(crc); + INIT_CRC32C(crc); + COMP_CRC32C(crc, &checkPoint, sizeof(checkPoint)); + COMP_CRC32C(crc, (char *) record, offsetof(XLogRecord, xl_crc)); + FIN_CRC32C(crc); record->xl_crc = crc; /* Create first XLOG segment file */ diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index f06daa2638f..da7ed92941c 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -684,8 +684,8 @@ ValidXLogRecord(XLogReaderState *state, XLogRecord *record, XLogRecPtr recptr) return false; } remaining -= SizeOfXLogRecord + len; - INIT_CRC32(crc); - COMP_CRC32(crc, XLogRecGetData(record), len); + INIT_CRC32C(crc); + COMP_CRC32C(crc, XLogRecGetData(record), len); /* Add in the backup blocks, if any */ blk = (char *) XLogRecGetData(record) + len; @@ -722,7 +722,7 @@ ValidXLogRecord(XLogReaderState *state, XLogRecord *record, XLogRecPtr recptr) return false; } remaining -= blen; - COMP_CRC32(crc, blk, blen); + COMP_CRC32C(crc, blk, blen); blk += blen; } @@ -736,10 +736,10 @@ ValidXLogRecord(XLogReaderState *state, XLogRecord *record, XLogRecPtr recptr) } /* Finally include the record header */ - COMP_CRC32(crc, (char *) record, offsetof(XLogRecord, xl_crc)); - FIN_CRC32(crc); + COMP_CRC32C(crc, (char *) record, offsetof(XLogRecord, xl_crc)); + FIN_CRC32C(crc); - if (!EQ_CRC32(record->xl_crc, crc)) + if (!EQ_CRC32C(record->xl_crc, crc)) { report_invalid_record(state, "incorrect resource manager data checksum in record at %X/%X", diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c index 5e59c6b819b..71c5fe2490a 100644 --- a/src/backend/replication/logical/snapbuild.c +++ b/src/backend/replication/logical/snapbuild.c @@ -1517,9 +1517,9 @@ SnapBuildSerialize(SnapBuild *builder, XLogRecPtr lsn) ondisk->magic = SNAPBUILD_MAGIC; ondisk->version = SNAPBUILD_VERSION; ondisk->length = needed_length; - INIT_CRC32(ondisk->checksum); - COMP_CRC32(ondisk->checksum, - ((char *) ondisk) + SnapBuildOnDiskNotChecksummedSize, + INIT_CRC32C(ondisk->checksum); + COMP_CRC32C(ondisk->checksum, + ((char *) ondisk) + SnapBuildOnDiskNotChecksummedSize, SnapBuildOnDiskConstantSize - SnapBuildOnDiskNotChecksummedSize); ondisk_c += sizeof(SnapBuildOnDisk); @@ -1531,20 +1531,20 @@ SnapBuildSerialize(SnapBuild *builder, XLogRecPtr lsn) ondisk->builder.running.xip = NULL; ondisk->builder.committed.xip = NULL; - COMP_CRC32(ondisk->checksum, - &ondisk->builder, - sizeof(SnapBuild)); + COMP_CRC32C(ondisk->checksum, + &ondisk->builder, + sizeof(SnapBuild)); /* copy running xacts */ sz = sizeof(TransactionId) * builder->running.xcnt_space; memcpy(ondisk_c, builder->running.xip, sz); - COMP_CRC32(ondisk->checksum, ondisk_c, sz); + COMP_CRC32C(ondisk->checksum, ondisk_c, sz); ondisk_c += sz; /* copy committed xacts */ sz = sizeof(TransactionId) * builder->committed.xcnt; memcpy(ondisk_c, builder->committed.xip, sz); - COMP_CRC32(ondisk->checksum, ondisk_c, sz); + COMP_CRC32C(ondisk->checksum, ondisk_c, sz); ondisk_c += sz; /* we have valid data now, open tempfile and write it there */ @@ -1672,8 +1672,8 @@ SnapBuildRestore(SnapBuild *builder, XLogRecPtr lsn) (errmsg("snapbuild state file \"%s\" has unsupported version %u instead of %u", path, ondisk.version, SNAPBUILD_VERSION))); - INIT_CRC32(checksum); - COMP_CRC32(checksum, + INIT_CRC32C(checksum); + COMP_CRC32C(checksum, ((char *) &ondisk) + SnapBuildOnDiskNotChecksummedSize, SnapBuildOnDiskConstantSize - SnapBuildOnDiskNotChecksummedSize); @@ -1687,7 +1687,7 @@ SnapBuildRestore(SnapBuild *builder, XLogRecPtr lsn) errmsg("could not read file \"%s\", read %d of %d: %m", path, readBytes, (int) sizeof(SnapBuild)))); } - COMP_CRC32(checksum, &ondisk.builder, sizeof(SnapBuild)); + COMP_CRC32C(checksum, &ondisk.builder, sizeof(SnapBuild)); /* restore running xacts information */ sz = sizeof(TransactionId) * ondisk.builder.running.xcnt_space; @@ -1701,7 +1701,7 @@ SnapBuildRestore(SnapBuild *builder, XLogRecPtr lsn) errmsg("could not read file \"%s\", read %d of %d: %m", path, readBytes, (int) sz))); } - COMP_CRC32(checksum, ondisk.builder.running.xip, sz); + COMP_CRC32C(checksum, ondisk.builder.running.xip, sz); /* restore committed xacts information */ sz = sizeof(TransactionId) * ondisk.builder.committed.xcnt; @@ -1715,12 +1715,12 @@ SnapBuildRestore(SnapBuild *builder, XLogRecPtr lsn) errmsg("could not read file \"%s\", read %d of %d: %m", path, readBytes, (int) sz))); } - COMP_CRC32(checksum, ondisk.builder.committed.xip, sz); + COMP_CRC32C(checksum, ondisk.builder.committed.xip, sz); CloseTransientFile(fd); /* verify checksum of what we've read */ - if (!EQ_CRC32(checksum, ondisk.checksum)) + if (!EQ_CRC32C(checksum, ondisk.checksum)) ereport(ERROR, (errcode_for_file_access(), errmsg("snapbuild state file %s: checksum mismatch, is %u, should be %u", diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c index f355f13d293..2c3ea555d09 100644 --- a/src/backend/replication/slot.c +++ b/src/backend/replication/slot.c @@ -993,7 +993,7 @@ SaveSlotToPath(ReplicationSlot *slot, const char *dir, int elevel) } cp.magic = SLOT_MAGIC; - INIT_CRC32(cp.checksum); + INIT_CRC32C(cp.checksum); cp.version = 1; cp.length = ReplicationSlotOnDiskDynamicSize; @@ -1003,9 +1003,9 @@ SaveSlotToPath(ReplicationSlot *slot, const char *dir, int elevel) SpinLockRelease(&slot->mutex); - COMP_CRC32(cp.checksum, - (char *) (&cp) + ReplicationSlotOnDiskConstantSize, - ReplicationSlotOnDiskDynamicSize); + COMP_CRC32C(cp.checksum, + (char *) (&cp) + ReplicationSlotOnDiskConstantSize, + ReplicationSlotOnDiskDynamicSize); if ((write(fd, &cp, sizeof(cp))) != sizeof(cp)) { @@ -1181,13 +1181,13 @@ RestoreSlotFromDisk(const char *name) CloseTransientFile(fd); - /* now verify the CRC32 */ - INIT_CRC32(checksum); - COMP_CRC32(checksum, - (char *) &cp + ReplicationSlotOnDiskConstantSize, - ReplicationSlotOnDiskDynamicSize); + /* now verify the CRC */ + INIT_CRC32C(checksum); + COMP_CRC32C(checksum, + (char *) &cp + ReplicationSlotOnDiskConstantSize, + ReplicationSlotOnDiskDynamicSize); - if (!EQ_CRC32(checksum, cp.checksum)) + if (!EQ_CRC32C(checksum, cp.checksum)) ereport(PANIC, (errmsg("replication slot file %s: checksum mismatch, is %u, should be %u", path, checksum, cp.checksum))); diff --git a/src/backend/utils/adt/tsgistidx.c b/src/backend/utils/adt/tsgistidx.c index 80039189c3f..e9cff4ffb69 100644 --- a/src/backend/utils/adt/tsgistidx.c +++ b/src/backend/utils/adt/tsgistidx.c @@ -201,9 +201,9 @@ gtsvector_compress(PG_FUNCTION_ARGS) { pg_crc32 c; - INIT_CRC32(c); - COMP_CRC32(c, words + ptr->pos, ptr->len); - FIN_CRC32(c); + INIT_LEGACY_CRC32(c); + COMP_LEGACY_CRC32(c, words + ptr->pos, ptr->len); + FIN_LEGACY_CRC32(c); *arr = *(int32 *) &c; arr++; diff --git a/src/backend/utils/adt/tsquery.c b/src/backend/utils/adt/tsquery.c index c8011a8ea92..5226bae2d17 100644 --- a/src/backend/utils/adt/tsquery.c +++ b/src/backend/utils/adt/tsquery.c @@ -280,9 +280,9 @@ pushValue(TSQueryParserState state, char *strval, int lenval, int16 weight, bool errmsg("word is too long in tsquery: \"%s\"", state->buffer))); - INIT_CRC32(valcrc); - COMP_CRC32(valcrc, strval, lenval); - FIN_CRC32(valcrc); + INIT_LEGACY_CRC32(valcrc); + COMP_LEGACY_CRC32(valcrc, strval, lenval); + FIN_LEGACY_CRC32(valcrc); pushValue_internal(state, valcrc, state->curop - state->op, lenval, weight, prefix); /* append the value string to state.op, enlarging buffer if needed first */ @@ -883,9 +883,9 @@ tsqueryrecv(PG_FUNCTION_ARGS) /* Looks valid. */ - INIT_CRC32(valcrc); - COMP_CRC32(valcrc, val, val_len); - FIN_CRC32(valcrc); + INIT_LEGACY_CRC32(valcrc); + COMP_LEGACY_CRC32(valcrc, val, val_len); + FIN_LEGACY_CRC32(valcrc); item->qoperand.weight = weight; item->qoperand.prefix = (prefix) ? true : false; diff --git a/src/backend/utils/cache/relmapper.c b/src/backend/utils/cache/relmapper.c index 95a2689fd42..b6b13308773 100644 --- a/src/backend/utils/cache/relmapper.c +++ b/src/backend/utils/cache/relmapper.c @@ -84,7 +84,7 @@ typedef struct RelMapFile int32 magic; /* always RELMAPPER_FILEMAGIC */ int32 num_mappings; /* number of valid RelMapping entries */ RelMapping mappings[MAX_MAPPINGS]; - int32 crc; /* CRC of all above */ + pg_crc32 crc; /* CRC of all above */ int32 pad; /* to make the struct size be 512 exactly */ } RelMapFile; @@ -673,11 +673,11 @@ load_relmap_file(bool shared) mapfilename))); /* verify the CRC */ - INIT_CRC32(crc); - COMP_CRC32(crc, (char *) map, offsetof(RelMapFile, crc)); - FIN_CRC32(crc); + INIT_CRC32C(crc); + COMP_CRC32C(crc, (char *) map, offsetof(RelMapFile, crc)); + FIN_CRC32C(crc); - if (!EQ_CRC32(crc, map->crc)) + if (!EQ_CRC32C(crc, map->crc)) ereport(FATAL, (errmsg("relation mapping file \"%s\" contains incorrect checksum", mapfilename))); @@ -719,9 +719,9 @@ write_relmap_file(bool shared, RelMapFile *newmap, if (newmap->num_mappings < 0 || newmap->num_mappings > MAX_MAPPINGS) elog(ERROR, "attempt to write bogus relation mapping"); - INIT_CRC32(newmap->crc); - COMP_CRC32(newmap->crc, (char *) newmap, offsetof(RelMapFile, crc)); - FIN_CRC32(newmap->crc); + INIT_CRC32C(newmap->crc); + COMP_CRC32C(newmap->crc, (char *) newmap, offsetof(RelMapFile, crc)); + FIN_CRC32C(newmap->crc); /* * Open the target file. We prefer to do this before entering the diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c index 32cc100fa25..b2e0793eb8f 100644 --- a/src/bin/pg_controldata/pg_controldata.c +++ b/src/bin/pg_controldata/pg_controldata.c @@ -178,13 +178,13 @@ main(int argc, char *argv[]) close(fd); /* Check the CRC. */ - INIT_CRC32(crc); - COMP_CRC32(crc, - (char *) &ControlFile, - offsetof(ControlFileData, crc)); - FIN_CRC32(crc); + INIT_CRC32C(crc); + COMP_CRC32C(crc, + (char *) &ControlFile, + offsetof(ControlFileData, crc)); + FIN_CRC32C(crc); - if (!EQ_CRC32(crc, ControlFile.crc)) + if (!EQ_CRC32C(crc, ControlFile.crc)) printf(_("WARNING: Calculated CRC checksum does not match value stored in file.\n" "Either the file is corrupt, or it has a different layout than this program\n" "is expecting. The results below are untrustworthy.\n\n")); diff --git a/src/bin/pg_resetxlog/pg_resetxlog.c b/src/bin/pg_resetxlog/pg_resetxlog.c index f4c1eafc67c..e224e67ad62 100644 --- a/src/bin/pg_resetxlog/pg_resetxlog.c +++ b/src/bin/pg_resetxlog/pg_resetxlog.c @@ -456,13 +456,13 @@ ReadControlFile(void) ((ControlFileData *) buffer)->pg_control_version == PG_CONTROL_VERSION) { /* Check the CRC. */ - INIT_CRC32(crc); - COMP_CRC32(crc, - buffer, - offsetof(ControlFileData, crc)); - FIN_CRC32(crc); + INIT_CRC32C(crc); + COMP_CRC32C(crc, + buffer, + offsetof(ControlFileData, crc)); + FIN_CRC32C(crc); - if (EQ_CRC32(crc, ((ControlFileData *) buffer)->crc)) + if (EQ_CRC32C(crc, ((ControlFileData *) buffer)->crc)) { /* Valid data... */ memcpy(&ControlFile, buffer, sizeof(ControlFile)); @@ -747,11 +747,11 @@ RewriteControlFile(void) ControlFile.xlog_seg_size = XLogSegSize; /* Contents are protected with a CRC */ - INIT_CRC32(ControlFile.crc); - COMP_CRC32(ControlFile.crc, - (char *) &ControlFile, - offsetof(ControlFileData, crc)); - FIN_CRC32(ControlFile.crc); + INIT_CRC32C(ControlFile.crc); + COMP_CRC32C(ControlFile.crc, + (char *) &ControlFile, + offsetof(ControlFileData, crc)); + FIN_CRC32C(ControlFile.crc); /* * We write out PG_CONTROL_SIZE bytes into pg_control, zero-padding the @@ -1032,10 +1032,10 @@ WriteEmptyXLOG(void) memcpy(XLogRecGetData(record), &ControlFile.checkPointCopy, sizeof(CheckPoint)); - INIT_CRC32(crc); - COMP_CRC32(crc, &ControlFile.checkPointCopy, sizeof(CheckPoint)); - COMP_CRC32(crc, (char *) record, offsetof(XLogRecord, xl_crc)); - FIN_CRC32(crc); + INIT_CRC32C(crc); + COMP_CRC32C(crc, &ControlFile.checkPointCopy, sizeof(CheckPoint)); + COMP_CRC32C(crc, (char *) record, offsetof(XLogRecord, xl_crc)); + FIN_CRC32C(crc); record->xl_crc = crc; /* Write the first page */ diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index 27b98995557..eaa1f9540e2 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -55,7 +55,7 @@ typedef struct BkpBlock /* * Each page of XLOG file has a header like this: */ -#define XLOG_PAGE_MAGIC 0xD07F /* can be used as WAL version indicator */ +#define XLOG_PAGE_MAGIC 0xD080 /* can be used as WAL version indicator */ typedef struct XLogPageHeaderData { diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 69ef30a3596..2c059c88c18 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201410071 +#define CATALOG_VERSION_NO 201411041 #endif diff --git a/src/include/utils/pg_crc.h b/src/include/utils/pg_crc.h index f43f4aad9dd..f871cba53f2 100644 --- a/src/include/utils/pg_crc.h +++ b/src/include/utils/pg_crc.h @@ -7,9 +7,22 @@ * A PAINLESS GUIDE TO CRC ERROR DETECTION ALGORITHMS, available from * http://www.ross.net/crc/ or several other net sites. * - * We use a normal (not "reflected", in Williams' terms) CRC, using initial - * all-ones register contents and a final bit inversion. + * We have three slightly different variants of a 32-bit CRC calculation: + * CRC-32C (Castagnoli polynomial), CRC-32 (Ethernet polynomial), and a legacy + * CRC-32 version that uses the lookup table in a funny way. They all consist + * of four macros: * + * INIT_(crc) + * Initialize a CRC accumulator + * + * COMP_(crc, data, len) + * Accumulate some (more) bytes into a CRC + * + * FIN_(crc) + * Finish a CRC calculation + * + * EQ_(c1, c2) + * Check for equality of two CRCs. * * Portions Copyright (c) 1996-2014, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California @@ -28,29 +41,81 @@ typedef uint32 pg_crc32; -/* Initialize a CRC accumulator */ -#define INIT_CRC32(crc) ((crc) = 0xFFFFFFFF) +/* + * CRC calculation using the CRC-32C (Castagnoli) polynomial. + * + * We use all-ones as the initial register contents and final bit inversion. + * This is the same algorithm used e.g. in iSCSI. See RFC 3385 for more + * details on the choice of polynomial. + */ +#define INIT_CRC32C(crc) ((crc) = 0xFFFFFFFF) +#define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) +#define COMP_CRC32C(crc, data, len) \ + COMP_CRC32_NORMAL_TABLE(crc, data, len, pg_crc32c_table) +#define EQ_CRC32C(c1, c2) ((c1) == (c2)) -/* Finish a CRC calculation */ -#define FIN_CRC32(crc) ((crc) ^= 0xFFFFFFFF) +/* + * CRC-32, the same used e.g. in Ethernet. + * + * This is currently only used in ltree and hstore contrib modules. It uses + * the same lookup table as the legacy algorithm below. New code should + * use the Castagnoli version instead. + */ +#define INIT_TRADITIONAL_CRC32(crc) ((crc) = 0xFFFFFFFF) +#define FIN_TRADITIONAL_CRC32(crc) ((crc) ^= 0xFFFFFFFF) +#define COMP_TRADITIONAL_CRC32(crc, data, len) \ + COMP_CRC32_NORMAL_TABLE(crc, data, len, pg_crc32_table) +#define EQ_TRADITIONAL_CRC32(c1, c2) ((c1) == (c2)) -/* Accumulate some (more) bytes into a CRC */ -#define COMP_CRC32(crc, data, len) \ -do { \ +/* + * The CRC algorithm used for WAL et al in pre-9.5 versions. + * + * This closely resembles the normal CRC-32 algorithm, but is subtly + * different. Using Williams' terms, we use the "normal" table, but with + * "reflected" code. That's bogus, but it was like that for years before + * anyone noticed. It does not correspond to any polynomial in a normal CRC + * algorithm, so it's not clear what the error-detection properties of this + * algorithm actually are. + * + * We still need to carry this around because it is used in a few on-disk + * structures that need to be pg_upgradeable. It should not be used in new + * code. + */ +#define INIT_LEGACY_CRC32(crc) ((crc) = 0xFFFFFFFF) +#define FIN_LEGACY_CRC32(crc) ((crc) ^= 0xFFFFFFFF) +#define COMP_LEGACY_CRC32(crc, data, len) \ + COMP_CRC32_REFLECTED_TABLE(crc, data, len, pg_crc32_table) +#define EQ_LEGACY_CRC32(c1, c2) ((c1) == (c2)) + +/* + * Common code for CRC computation using a lookup table. + */ +#define COMP_CRC32_NORMAL_TABLE(crc, data, len, table) \ +do { \ const unsigned char *__data = (const unsigned char *) (data); \ uint32 __len = (len); \ \ while (__len-- > 0) \ { \ - int __tab_index = ((int) ((crc) >> 24) ^ *__data++) & 0xFF; \ - (crc) = pg_crc32_table[__tab_index] ^ ((crc) << 8); \ + int __tab_index = ((int) (crc) ^ *__data++) & 0xFF; \ + (crc) = table[__tab_index] ^ ((crc) >> 8); \ } \ } while (0) -/* Check for equality of two CRCs */ -#define EQ_CRC32(c1,c2) ((c1) == (c2)) +#define COMP_CRC32_REFLECTED_TABLE(crc, data, len, table) \ +do { \ + const unsigned char *__data = (const unsigned char *) (data); \ + uint32 __len = (len); \ +\ + while (__len-- > 0) \ + { \ + int __tab_index = ((int) ((crc) >> 24) ^ *__data++) & 0xFF; \ + (crc) = table[__tab_index] ^ ((crc) << 8); \ + } \ +} while (0) -/* Constant table for CRC calculation */ +/* Constant tables for CRC-32C and CRC-32 polynomials */ +extern CRCDLLIMPORT const uint32 pg_crc32c_table[]; extern CRCDLLIMPORT const uint32 pg_crc32_table[]; #endif /* PG_CRC_H */ diff --git a/src/include/utils/pg_crc_tables.h b/src/include/utils/pg_crc_tables.h index f56613776dc..cb6b470885d 100644 --- a/src/include/utils/pg_crc_tables.h +++ b/src/include/utils/pg_crc_tables.h @@ -11,9 +11,8 @@ * A PAINLESS GUIDE TO CRC ERROR DETECTION ALGORITHMS, available from * http://www.ross.net/crc/download/crc_v3.txt or several other net sites. * - * We use a normal (not "reflected", in Williams' terms) CRC, using initial - * all-ones register contents and a final bit inversion. - * + * These lookup tables are for normal, not "reflected", in Williams' terms, + * CRC. * * Portions Copyright (c) 1996-2014, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California @@ -25,6 +24,78 @@ #ifndef PG_CRC_TABLES_H #define PG_CRC_TABLES_H +/* + * This table is based on the so-called Castagnoli polynomial (the same + * that is used e.g. in iSCSI). + */ +const uint32 pg_crc32c_table[256] = { + 0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4, + 0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB, + 0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B, + 0x4D43CFD0, 0xBF284CD3, 0xAC78BF27, 0x5E133C24, + 0x105EC76F, 0xE235446C, 0xF165B798, 0x030E349B, + 0xD7C45070, 0x25AFD373, 0x36FF2087, 0xC494A384, + 0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, 0x89D76C54, + 0x5D1D08BF, 0xAF768BBC, 0xBC267848, 0x4E4DFB4B, + 0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, 0x33ED7D2A, + 0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35, + 0xAA64D611, 0x580F5512, 0x4B5FA6E6, 0xB93425E5, + 0x6DFE410E, 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA, + 0x30E349B1, 0xC288CAB2, 0xD1D83946, 0x23B3BA45, + 0xF779DEAE, 0x05125DAD, 0x1642AE59, 0xE4292D5A, + 0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A, + 0x7DA08661, 0x8FCB0562, 0x9C9BF696, 0x6EF07595, + 0x417B1DBC, 0xB3109EBF, 0xA0406D4B, 0x522BEE48, + 0x86E18AA3, 0x748A09A0, 0x67DAFA54, 0x95B17957, + 0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687, + 0x0C38D26C, 0xFE53516F, 0xED03A29B, 0x1F682198, + 0x5125DAD3, 0xA34E59D0, 0xB01EAA24, 0x42752927, + 0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38, + 0xDBFC821C, 0x2997011F, 0x3AC7F2EB, 0xC8AC71E8, + 0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7, + 0x61C69362, 0x93AD1061, 0x80FDE395, 0x72966096, + 0xA65C047D, 0x5437877E, 0x4767748A, 0xB50CF789, + 0xEB1FCBAD, 0x197448AE, 0x0A24BB5A, 0xF84F3859, + 0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46, + 0x7198540D, 0x83F3D70E, 0x90A324FA, 0x62C8A7F9, + 0xB602C312, 0x44694011, 0x5739B3E5, 0xA55230E6, + 0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36, + 0x3CDB9BDD, 0xCEB018DE, 0xDDE0EB2A, 0x2F8B6829, + 0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C, + 0x456CAC67, 0xB7072F64, 0xA457DC90, 0x563C5F93, + 0x082F63B7, 0xFA44E0B4, 0xE9141340, 0x1B7F9043, + 0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C, + 0x92A8FC17, 0x60C37F14, 0x73938CE0, 0x81F80FE3, + 0x55326B08, 0xA759E80B, 0xB4091BFF, 0x466298FC, + 0x1871A4D8, 0xEA1A27DB, 0xF94AD42F, 0x0B21572C, + 0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033, + 0xA24BB5A6, 0x502036A5, 0x4370C551, 0xB11B4652, + 0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D, + 0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, 0x3BC21E9D, + 0xEF087A76, 0x1D63F975, 0x0E330A81, 0xFC588982, + 0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D, + 0x758FE5D6, 0x87E466D5, 0x94B49521, 0x66DF1622, + 0x38CC2A06, 0xCAA7A905, 0xD9F75AF1, 0x2B9CD9F2, + 0xFF56BD19, 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED, + 0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530, + 0x0417B1DB, 0xF67C32D8, 0xE52CC12C, 0x1747422F, + 0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, 0x5A048DFF, + 0x8ECEE914, 0x7CA56A17, 0x6FF599E3, 0x9D9E1AE0, + 0xD3D3E1AB, 0x21B862A8, 0x32E8915C, 0xC083125F, + 0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540, + 0x590AB964, 0xAB613A67, 0xB831C993, 0x4A5A4A90, + 0x9E902E7B, 0x6CFBAD78, 0x7FAB5E8C, 0x8DC0DD8F, + 0xE330A81A, 0x115B2B19, 0x020BD8ED, 0xF0605BEE, + 0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, 0x37FACCF1, + 0x69E9F0D5, 0x9B8273D6, 0x88D28022, 0x7AB90321, + 0xAE7367CA, 0x5C18E4C9, 0x4F48173D, 0xBD23943E, + 0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81, + 0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E, + 0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E, + 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351 +}; + + /* * This table is based on the polynomial * x^32+x^26+x^23+x^22+x^16+x^12+x^11+x^10+x^8+x^7+x^5+x^4+x^2+x+1.