xfs: optimise CRC updates

Nick Piggin reported that the CRC overhead in an fsync heavy
workload was higher than expected on a Power8 machine. Part of this
was to do with the fact that the power8 CRC implementation is not
efficient for CRC lengths of less than 512 bytes, and so the way we
split the CRCs over the CRC field means a lot of the CRCs are
reduced to being less than than optimal size.

To optimise this, change the CRC update mechanism to zero the CRC
field first, and then compute the CRC in one pass over the buffer
and write the result back into the buffer. We can do this safely
because anything writing a CRC has exclusive access to the buffer
the CRC is being calculated over.

We leave the CRC verify code the same - it still splits the CRC
calculation - because we do not want read-only operations modifying
the underlying buffer. This is because read-only operations may not
have an exclusive access to the buffer guaranteed, and so temporary
modifications could leak out to to other processes accessing the
buffer concurrently.

Signed-off-by: Dave Chinner <dchinner@redhat.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Dave Chinner <david@fromorbit.com>

diff --git a/fs/xfs/libxfs/xfs_cksum.h b/fs/xfs/libxfs/xfs_cksum.h
index fad1676ad8cd25cd892799cee07cd01d3009fdd0..a416c7cb23ea57ed44df6db0844f623beab1e4b7 100644 (file)
--- a/fs/xfs/libxfs/xfs_cksum.h
+++ b/fs/xfs/libxfs/xfs_cksum.h
@@ -6,10 +6,11 @@
 /*
  * Calculate the intermediate checksum for a buffer that has the CRC field
  * inside it.  The offset of the 32bit crc fields is passed as the
- * cksum_offset parameter.
+ * cksum_offset parameter. We do not modify the buffer during verification,
+ * hence we have to split the CRC calculation across the cksum_offset.
  */
 static inline __uint32_t
-xfs_start_cksum(char *buffer, size_t length, unsigned long cksum_offset)
+xfs_start_cksum_safe(char *buffer, size_t length, unsigned long cksum_offset)
 {
        __uint32_t zero = 0;
        __uint32_t crc;
@@ -25,6 +26,20 @@ xfs_start_cksum(char *buffer, size_t length, unsigned long cksum_offset)
                      length - (cksum_offset + sizeof(__be32)));
 }
 
+/*
+ * Fast CRC method where the buffer is modified. Callers must have exclusive
+ * access to the buffer while the calculation takes place.
+ */
+static inline __uint32_t
+xfs_start_cksum_update(char *buffer, size_t length, unsigned long cksum_offset)
+{
+       /* zero the CRC field */
+       *(__le32 *)(buffer + cksum_offset) = 0;
+
+       /* single pass CRC calculation for the entire buffer */
+       return crc32c(XFS_CRC_SEED, buffer, length);
+}
+
 /*
  * Convert the intermediate checksum to the final ondisk format.
  *
@@ -40,11 +55,14 @@ xfs_end_cksum(__uint32_t crc)
 
 /*
  * Helper to generate the checksum for a buffer.
+ *
+ * This modifies the buffer temporarily - callers must have exclusive
+ * access to the buffer while the calculation takes place.
  */
 static inline void
 xfs_update_cksum(char *buffer, size_t length, unsigned long cksum_offset)
 {
-       __uint32_t crc = xfs_start_cksum(buffer, length, cksum_offset);
+       __uint32_t crc = xfs_start_cksum_update(buffer, length, cksum_offset);
 
        *(__le32 *)(buffer + cksum_offset) = xfs_end_cksum(crc);
 }
@@ -55,7 +73,7 @@ xfs_update_cksum(char *buffer, size_t length, unsigned long cksum_offset)
 static inline int
 xfs_verify_cksum(char *buffer, size_t length, unsigned long cksum_offset)
 {
-       __uint32_t crc = xfs_start_cksum(buffer, length, cksum_offset);
+       __uint32_t crc = xfs_start_cksum_safe(buffer, length, cksum_offset);
 
        return *(__le32 *)(buffer + cksum_offset) == xfs_end_cksum(crc);
 }

diff --git a/fs/xfs/libxfs/xfs_inode_buf.c b/fs/xfs/libxfs/xfs_inode_buf.c
index c906e50515f031f0806f540d7792ef69e4168fb2..9004665d679a3b9873ac15436c2c2f0513ab8c9e 100644 (file)
--- a/fs/xfs/libxfs/xfs_inode_buf.c
+++ b/fs/xfs/libxfs/xfs_inode_buf.c
@@ -444,7 +444,7 @@ xfs_dinode_calc_crc(
                return;
 
        ASSERT(xfs_sb_version_hascrc(&mp->m_sb));
-       crc = xfs_start_cksum((char *)dip, mp->m_sb.sb_inodesize,
+       crc = xfs_start_cksum_update((char *)dip, mp->m_sb.sb_inodesize,
                              XFS_DINODE_CRC_OFF);
        dip->di_crc = xfs_end_cksum(crc);
 }

diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c
index 3b74fa011bb1566ce43dedc2fbdbc49336645d84..3ebe444eb60fa526ce30032371fb8bccfb6fd951 100644 (file)
--- a/fs/xfs/xfs_log.c
+++ b/fs/xfs/xfs_log.c
@@ -1668,7 +1668,7 @@ xlog_cksum(
        __uint32_t              crc;
 
        /* first generate the crc for the record header ... */
-       crc = xfs_start_cksum((char *)rhead,
+       crc = xfs_start_cksum_update((char *)rhead,
                              sizeof(struct xlog_rec_header),
                              offsetof(struct xlog_rec_header, h_crc));
 

diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c
index 9b3d7c76915d9c92948bed063d79bd10a6107a77..56b7a2f6aaf28eab1608cec6c702bef01702c577 100644 (file)
--- a/fs/xfs/xfs_log_recover.c
+++ b/fs/xfs/xfs_log_recover.c
@@ -5113,19 +5113,21 @@ xlog_recover_process(
        struct list_head        *buffer_list)
 {
        int                     error;
+       __le32                  old_crc = rhead->h_crc;
        __le32                  crc;
 
+
        crc = xlog_cksum(log, rhead, dp, be32_to_cpu(rhead->h_len));
 
        /*
         * Nothing else to do if this is a CRC verification pass. Just return
         * if this a record with a non-zero crc. Unfortunately, mkfs always
-        * sets h_crc to 0 so we must consider this valid even on v5 supers.
+        * sets old_crc to 0 so we must consider this valid even on v5 supers.
         * Otherwise, return EFSBADCRC on failure so the callers up the stack
         * know precisely what failed.
         */
        if (pass == XLOG_RECOVER_CRCPASS) {
-               if (rhead->h_crc && crc != rhead->h_crc)
+               if (old_crc && crc != old_crc)
                        return -EFSBADCRC;
                return 0;
        }
@@ -5136,11 +5138,11 @@ xlog_recover_process(
         * zero CRC check prevents warnings from being emitted when upgrading
         * the kernel from one that does not add CRCs by default.
         */
-       if (crc != rhead->h_crc) {
-               if (rhead->h_crc || xfs_sb_version_hascrc(&log->l_mp->m_sb)) {
+       if (crc != old_crc) {
+               if (old_crc || xfs_sb_version_hascrc(&log->l_mp->m_sb)) {
                        xfs_alert(log->l_mp,
                "log record CRC mismatch: found 0x%x, expected 0x%x.",
-                                       le32_to_cpu(rhead->h_crc),
+                                       le32_to_cpu(old_crc),
                                        le32_to_cpu(crc));
                        xfs_hex_dump(dp, 32);
                }