ARM: 8297/1: cache-l2x0: optimize aurora range operations

The aurora_inv_range(), aurora_clean_range() and aurora_flush_range()
functions are highly redundant, both in source and in object code, and
they are harder to understand than necessary.

By moving the range loop into the aurora_pa_range() function, they
become trivial wrappers, and the object code start looking like what
one would expect for an optimal implementation.

Further optimization may be possible by using the per-CPU "virtual"
registers to avoid the spinlocks in most cases.

 (on Armada 370 RD and Armada XP GP, boot tested, plus a little bit of
 DMA traffic by reading data from a SD card)

Reviewed-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
Tested-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>

diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c
index 404c598da27d9b6e5f32e3c3ffa3825fb099928f..5ea2d6d417f715408dad81b0f53d320e0fc7aa4f 100644 (file)
--- a/arch/arm/mm/cache-l2x0.c
+++ b/arch/arm/mm/cache-l2x0.c
@@ -1256,7 +1256,7 @@ static const struct l2c_init_data of_l2c310_coherent_data __initconst = {
  * noninclusive, while the hardware cache range operations use
  * inclusive start and end addresses.
  */
-static unsigned long calc_range_end(unsigned long start, unsigned long end)
+static unsigned long aurora_range_end(unsigned long start, unsigned long end)
 {
        /*
         * Limit the number of cache lines processed at once,
@@ -1275,26 +1275,13 @@ static unsigned long calc_range_end(unsigned long start, unsigned long end)
        return end;
 }
 
-/*
- * Make sure 'start' and 'end' reference the same page, as L2 is PIPT
- * and range operations only do a TLB lookup on the start address.
- */
 static void aurora_pa_range(unsigned long start, unsigned long end,
-                       unsigned long offset)
+                           unsigned long offset)
 {
        void __iomem *base = l2x0_base;
+       unsigned long range_end;
        unsigned long flags;
 
-       raw_spin_lock_irqsave(&l2x0_lock, flags);
-       writel_relaxed(start, base + AURORA_RANGE_BASE_ADDR_REG);
-       writel_relaxed(end, base + offset);
-       raw_spin_unlock_irqrestore(&l2x0_lock, flags);
-
-       writel_relaxed(0, base + AURORA_SYNC_REG);
-}
-
-static void aurora_inv_range(unsigned long start, unsigned long end)
-{
        /*
         * round start and end adresses up to cache line size
         */
@@ -1302,15 +1289,24 @@ static void aurora_inv_range(unsigned long start, unsigned long end)
        end = ALIGN(end, CACHE_LINE_SIZE);
 
        /*
-        * Invalidate all full cache lines between 'start' and 'end'.
+        * perform operation on all full cache lines between 'start' and 'end'
         */
        while (start < end) {
-               unsigned long range_end = calc_range_end(start, end);
-               aurora_pa_range(start, range_end - CACHE_LINE_SIZE,
-                               AURORA_INVAL_RANGE_REG);
+               range_end = aurora_range_end(start, end);
+
+               raw_spin_lock_irqsave(&l2x0_lock, flags);
+               writel_relaxed(start, base + AURORA_RANGE_BASE_ADDR_REG);
+               writel_relaxed(range_end - CACHE_LINE_SIZE, base + offset);
+               raw_spin_unlock_irqrestore(&l2x0_lock, flags);
+
+               writel_relaxed(0, base + AURORA_SYNC_REG);
                start = range_end;
        }
 }
+static void aurora_inv_range(unsigned long start, unsigned long end)
+{
+       aurora_pa_range(start, end, AURORA_INVAL_RANGE_REG);
+}
 
 static void aurora_clean_range(unsigned long start, unsigned long end)
 {
@@ -1318,36 +1314,16 @@ static void aurora_clean_range(unsigned long start, unsigned long end)
         * If L2 is forced to WT, the L2 will always be clean and we
         * don't need to do anything here.
         */
-       if (!l2_wt_override) {
-               start &= ~(CACHE_LINE_SIZE - 1);
-               end = ALIGN(end, CACHE_LINE_SIZE);
-               while (start != end) {
-                       unsigned long range_end = calc_range_end(start, end);
-                       aurora_pa_range(start, range_end - CACHE_LINE_SIZE,
-                                       AURORA_CLEAN_RANGE_REG);
-                       start = range_end;
-               }
-       }
+       if (!l2_wt_override)
+               aurora_pa_range(start, end, AURORA_CLEAN_RANGE_REG);
 }
 
 static void aurora_flush_range(unsigned long start, unsigned long end)
 {
-       start &= ~(CACHE_LINE_SIZE - 1);
-       end = ALIGN(end, CACHE_LINE_SIZE);
-       while (start != end) {
-               unsigned long range_end = calc_range_end(start, end);
-               /*
-                * If L2 is forced to WT, the L2 will always be clean and we
-                * just need to invalidate.
-                */
-               if (l2_wt_override)
-                       aurora_pa_range(start, range_end - CACHE_LINE_SIZE,
-                                                       AURORA_INVAL_RANGE_REG);
-               else
-                       aurora_pa_range(start, range_end - CACHE_LINE_SIZE,
-                                                       AURORA_FLUSH_RANGE_REG);
-               start = range_end;
-       }
+       if (l2_wt_override)
+               aurora_pa_range(start, end, AURORA_INVAL_RANGE_REG);
+       else
+               aurora_pa_range(start, end, AURORA_FLUSH_RANGE_REG);
 }
 
 static void aurora_flush_all(void)