Merge tag 'v3.10.55' into update

[GitHub/mt8127/android_kernel_alcatel_ttab.git] / arch / arm / kernel / topology.c

/*
 * arch/arm/kernel/topology.c
 *
 * Copyright (C) 2011 Linaro Limited.
 * Written by: Vincent Guittot
 *
 * based on arch/sh/kernel/topology.c
 *
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 */

#include <linux/cpu.h>
#include <linux/cpumask.h>
#include <linux/export.h>
#include <linux/init.h>
#include <linux/percpu.h>
#include <linux/node.h>
#include <linux/nodemask.h>
#include <linux/of.h>
#include <linux/sched.h>
#include <linux/slab.h>

#include <asm/cputype.h>
#include <asm/smp_plat.h>
#include <asm/topology.h>

/*
 * cpu power scale management
 */

/*
 * cpu power table
 * This per cpu data structure describes the relative capacity of each core.
 * On a heteregenous system, cores don't have the same computation capacity
 * and we reflect that difference in the cpu_power field so the scheduler can
 * take this difference into account during load balance. A per cpu structure
 * is preferred because each CPU updates its own cpu_power field during the
 * load balance except for idle cores. One idle core is selected to run the
 * rebalance_domains for all idle cores and the cpu_power can be updated
 * during this sequence.
 */

/* when CONFIG_ARCH_SCALE_INVARIANT_CPU_CAPACITY is in use, a new measure of
 * compute capacity is available. This is limited to a maximum of 1024 and
 * scaled between 0 and 1023 according to frequency.
 * Cores with different base CPU powers are scaled in line with this.
 * CPU capacity for each core represents a comparable ratio to maximum
 * achievable core compute capacity for a core in this system.
 *
 * e.g.1 If all cores in the system have a base CPU power of 1024 according to
 * efficiency calculations and are DVFS scalable between 500MHz and 1GHz, the
 * cores currently at 1GHz will have CPU power of 1024 whilst the cores
 * currently at 500MHz will have CPU power of 512.
 *
 * e.g.2
 * If core 0 has a base CPU power of 2048 and runs at 500MHz & 1GHz whilst
 * core 1 has a base CPU power of 1024 and runs at 100MHz and 200MHz, then
 * the following possibilities are available:
 *
 * cpu power\| 1GHz:100Mhz | 1GHz : 200MHz | 500MHz:100MHz | 500MHz:200MHz |
 * ----------|-------------|---------------|---------------|---------------|
 *    core 0 |    1024     |     1024      |     512       |     512       |
 *    core 1 |     256     |      512      |     256       |     512       |
 *
 * This information may be useful to the scheduler when load balancing,
 * so that the compute capacity of the core a task ran on can be baked into
 * task load histories.
 */
static DEFINE_PER_CPU(unsigned long, cpu_scale);
#ifdef CONFIG_ARCH_SCALE_INVARIANT_CPU_CAPACITY
static DEFINE_PER_CPU(unsigned long, base_cpu_capacity);
static DEFINE_PER_CPU(unsigned long, invariant_cpu_capacity);
static DEFINE_PER_CPU(unsigned long, prescaled_cpu_capacity);
#endif /* CONFIG_ARCH_SCALE_INVARIANT_CPU_CAPACITY */

static int frequency_invariant_power_enabled = 1;

/* >0=1, <=0=0 */
void arch_set_invariant_power_enabled(int val)
{
        if(val>0)
                frequency_invariant_power_enabled = 1;
        else
                frequency_invariant_power_enabled = 0;
}

int arch_get_invariant_power_enabled(void)
{
        return frequency_invariant_power_enabled;
}

unsigned long arch_scale_freq_power(struct sched_domain *sd, int cpu)
{
        return per_cpu(cpu_scale, cpu);
}

#ifdef CONFIG_ARCH_SCALE_INVARIANT_CPU_CAPACITY
unsigned long arch_get_cpu_capacity(int cpu)
{
        return per_cpu(invariant_cpu_capacity, cpu);
}
unsigned long arch_get_max_cpu_capacity(int cpu)
{
        return per_cpu(base_cpu_capacity, cpu);
}
#endif /* CONFIG_ARCH_SCALE_INVARIANT_CPU_CAPACITY */

static void set_power_scale(unsigned int cpu, unsigned long power)
{
        per_cpu(cpu_scale, cpu) = power;
}

#ifdef CONFIG_OF
struct cpu_efficiency {
        const char *compatible;
        unsigned long efficiency;
};

/*
 * Table of relative efficiency of each processors
 * The efficiency value must fit in 20bit and the final
 * cpu_scale value must be in the range
 *   0 < cpu_scale < 3*SCHED_POWER_SCALE/2
 * in order to return at most 1 when DIV_ROUND_CLOSEST
 * is used to compute the capacity of a CPU.
 * Processors that are not defined in the table,
 * use the default SCHED_POWER_SCALE value for cpu_scale.
 */
struct cpu_efficiency table_efficiency[] = {
        {"arm,cortex-a15", 3891},
        {"arm,cortex-a17", 3276},
        {"arm,cortex-a12", 3276},
        {"arm,cortex-a53", 2520},
        {"arm,cortex-a7",  2048},
        {NULL, },
};

struct cpu_capacity {
        unsigned long hwid;
        unsigned long capacity;
};

struct cpu_capacity *cpu_capacity;

unsigned long middle_capacity = 1;
/*
 * Iterate all CPUs' descriptor in DT and compute the efficiency
 * (as per table_efficiency). Also calculate a middle efficiency
 * as close as possible to  (max{eff_i} - min{eff_i}) / 2
 * This is later used to scale the cpu_power field such that an
 * 'average' CPU is of middle power. Also see the comments near
 * table_efficiency[] and update_cpu_power().
 */
static void __init parse_dt_topology(void)
{
        struct cpu_efficiency *cpu_eff;
        struct device_node *cn = NULL;
        unsigned long min_capacity = (unsigned long)(-1);
        unsigned long max_capacity = 0;
        unsigned long capacity = 0;
        int alloc_size, cpu = 0;

        alloc_size = nr_cpu_ids * sizeof(struct cpu_capacity);
        cpu_capacity = kzalloc(alloc_size, GFP_NOWAIT);

        while ((cn = of_find_node_by_type(cn, "cpu"))) {
                const u32 *rate, *reg;
                int len;

                if (cpu >= num_possible_cpus())
                        break;

                for (cpu_eff = table_efficiency; cpu_eff->compatible; cpu_eff++)
                        if (of_device_is_compatible(cn, cpu_eff->compatible))
                                break;

                if (cpu_eff->compatible == NULL)
                        continue;

                rate = of_get_property(cn, "clock-frequency", &len);
                if (!rate || len != 4) {
                        pr_err("%s missing clock-frequency property\n",
                                   cn->full_name);
                        continue;
                }

                reg = of_get_property(cn, "reg", &len);
                if (!reg || len != 4) {
                        pr_err("%s missing reg property\n", cn->full_name);
                        continue;
                }

                capacity = ((be32_to_cpup(rate)) >> 20) * cpu_eff->efficiency;

                /* Save min capacity of the system */
                if (capacity < min_capacity)
                        min_capacity = capacity;

                /* Save max capacity of the system */
                if (capacity > max_capacity)
                        max_capacity = capacity;

                cpu_capacity[cpu].capacity = capacity;
                cpu_capacity[cpu++].hwid = be32_to_cpup(reg);
        }

        if (cpu < num_possible_cpus())
                cpu_capacity[cpu].hwid = (unsigned long)(-1);

        /* If min and max capacities are equals, we bypass the update of the
         * cpu_scale because all CPUs have the same capacity. Otherwise, we
         * compute a middle_capacity factor that will ensure that the capacity
         * of an 'average' CPU of the system will be as close as possible to
         * SCHED_POWER_SCALE, which is the default value, but with the
         * constraint explained near table_efficiency[].
         */
        if (min_capacity == max_capacity)
                cpu_capacity[0].hwid = (unsigned long)(-1);
        else if (4*max_capacity < (3*(max_capacity + min_capacity)))
                middle_capacity = (min_capacity + max_capacity)
                        >> (SCHED_POWER_SHIFT+1);
        else
                middle_capacity = ((max_capacity / 3)
                                                   >> (SCHED_POWER_SHIFT-1)) + 1;

}

/*
 * Look for a customed capacity of a CPU in the cpu_capacity table during the
 * boot. The update of all CPUs is in O(n^2) for heteregeneous system but the
 * function returns directly for SMP system.
 */
void update_cpu_power(unsigned int cpu, unsigned long hwid)
{
        unsigned int idx = 0;

        /* look for the cpu's hwid in the cpu capacity table */
        for (idx = 0; idx < num_possible_cpus(); idx++) {
                if (cpu_capacity[idx].hwid == hwid)
                        break;

                if (cpu_capacity[idx].hwid == -1)
                        return;
        }

        if (idx == num_possible_cpus())
                return;

        set_power_scale(cpu, cpu_capacity[idx].capacity / middle_capacity);

        printk(KERN_INFO "CPU%u: update cpu_power %lu\n",
                   cpu, arch_scale_freq_power(NULL, cpu));
}

#else
static inline void parse_dt_topology(void) {}
static inline void update_cpu_power(unsigned int cpuid, unsigned int mpidr) {}
#endif

/*
 * cpu topology table
 */
struct cputopo_arm cpu_topology[NR_CPUS];
EXPORT_SYMBOL_GPL(cpu_topology);

#if defined (CONFIG_MTK_SCHED_CMP_PACK_SMALL_TASK) || defined (CONFIG_HMP_PACK_SMALL_TASK)
int arch_sd_share_power_line(void)
{
        return 0*SD_SHARE_POWERLINE;
}
#endif /* CONFIG_MTK_SCHED_CMP_PACK_SMALL_TASK || CONFIG_HMP_PACK_SMALL_TASK  */

const struct cpumask *cpu_coregroup_mask(int cpu)
{
        return &cpu_topology[cpu].core_sibling;
}

void update_siblings_masks(unsigned int cpuid)
{
        struct cputopo_arm *cpu_topo, *cpuid_topo = &cpu_topology[cpuid];
        int cpu;

        /* update core and thread sibling masks */
        for_each_possible_cpu(cpu) {
                cpu_topo = &cpu_topology[cpu];

                if (cpuid_topo->socket_id != cpu_topo->socket_id)
                        continue;

                cpumask_set_cpu(cpuid, &cpu_topo->core_sibling);
                if (cpu != cpuid)
                        cpumask_set_cpu(cpu, &cpuid_topo->core_sibling);

                if (cpuid_topo->core_id != cpu_topo->core_id)
                        continue;

                cpumask_set_cpu(cpuid, &cpu_topo->thread_sibling);
                if (cpu != cpuid)
                        cpumask_set_cpu(cpu, &cpuid_topo->thread_sibling);
        }
        smp_wmb();
}

#ifdef CONFIG_MTK_CPU_TOPOLOGY

enum {
        ARCH_UNKNOWN = 0,
        ARCH_SINGLE_CLUSTER,
        ARCH_MULTI_CLUSTER,
        ARCH_BIG_LITTLE,
};

struct cpu_cluster {
        int cluster_id;
        cpumask_t siblings;
        void *next;
};

struct cpu_compatible {
        const char *name;
        const unsigned int cpuidr;
        struct cpu_cluster *cluster;
        int clscnt;
};

struct cpu_arch_info {
        struct cpu_compatible *compat_big;
        struct cpu_compatible *compat_ltt;
        bool arch_ready;
        int arch_type;
        int nr_clusters;
};

/* NOTE: absolute decending ordered by cpu capacity */
struct cpu_compatible cpu_compat_table[] = {
        { "arm,cortex-a15", ARM_CPU_PART_CORTEX_A15, NULL, 0 },
        { "arm,cortex-a17", ARM_CPU_PART_CORTEX_A17, NULL, 0 },
        { "arm,cortex-a12", ARM_CPU_PART_CORTEX_A12, NULL, 0 },
        { "arm,cortex-a53", ARM_CPU_PART_CORTEX_A53, NULL, 0 },
        { "arm,cortex-a9",  ARM_CPU_PART_CORTEX_A9, NULL, 0 },
        { "arm,cortex-a7",  ARM_CPU_PART_CORTEX_A7, NULL, 0 },
        { NULL, 0, NULL, 0 }
};

static struct cpu_compatible* compat_cputopo[NR_CPUS];

static struct cpu_arch_info default_cpu_arch = {
        NULL,
        NULL,
        0,
        ARCH_UNKNOWN,
        0,
};
static struct cpu_arch_info *glb_cpu_arch = &default_cpu_arch;

static int __arch_type(void)
{
        int i, num_compat = 0;

        if (!glb_cpu_arch->arch_ready)
                return ARCH_UNKNOWN;

        // return the cached setting if query more than once.
        if (glb_cpu_arch->arch_type != ARCH_UNKNOWN)
                return glb_cpu_arch->arch_type;

        for (i = 0; i < ARRAY_SIZE(cpu_compat_table); i++) {
                struct cpu_compatible *mc = &cpu_compat_table[i];
                if (mc->clscnt != 0)
                        num_compat++;
        }

        if (num_compat > 1)
                glb_cpu_arch->arch_type = ARCH_BIG_LITTLE;
        else if (glb_cpu_arch->nr_clusters > 1)
                glb_cpu_arch->arch_type = ARCH_MULTI_CLUSTER;
        else if (num_compat == 1 && glb_cpu_arch->nr_clusters == 1)
                glb_cpu_arch->arch_type = ARCH_SINGLE_CLUSTER;

        return glb_cpu_arch->arch_type;
}

static DEFINE_SPINLOCK(__cpu_cluster_lock);
static void __setup_cpu_cluster(const unsigned int cpu,
                                                                struct cpu_compatible * const cpt,
                                                                const u32 mpidr)
{
        struct cpu_cluster *prev_cls, *cls;
        u32 cls_id = -1;

        if (mpidr & MPIDR_MT_BITMASK)
                cls_id = MPIDR_AFFINITY_LEVEL(mpidr, 2);
        else
                cls_id = MPIDR_AFFINITY_LEVEL(mpidr, 1);

        spin_lock(&__cpu_cluster_lock);

        cls = cpt->cluster;
        prev_cls = cls;
        while (cls) {
                if (cls->cluster_id == cls_id)
                        break;
                prev_cls = cls;
                cls = (struct cpu_cluster *)cls->next;
        }

        if (!cls) {
                cls = kzalloc(sizeof(struct cpu_cluster), GFP_ATOMIC);
                BUG_ON(!cls);
                cls->cluster_id = cls_id;
                cpt->clscnt++;
                glb_cpu_arch->nr_clusters++;
                /* link it */
                if (!cpt->cluster)
                        cpt->cluster = cls;
                else
                        prev_cls->next = cls;
        }
        BUG_ON(cls->cluster_id != cls_id);

        cpumask_set_cpu(cpu, &cls->siblings);
        smp_wmb();

        spin_unlock(&__cpu_cluster_lock);
}

static void setup_cputopo(const unsigned int cpu,
                                                  struct cpu_compatible * const cpt,
                                                  const u32 mpidr)

{
        if (compat_cputopo[cpu])
                return;

        compat_cputopo[cpu] = cpt;

        if (!glb_cpu_arch->compat_big || glb_cpu_arch->compat_big > cpt)
                glb_cpu_arch->compat_big = cpt;

        if (!glb_cpu_arch->compat_ltt || glb_cpu_arch->compat_ltt < cpt)
                glb_cpu_arch->compat_ltt = cpt;

        __setup_cpu_cluster(cpu, cpt, mpidr);
}

static void setup_cputopo_def(const unsigned int cpu)
{
        struct cpu_compatible *idx = NULL;
        unsigned int cpuidr = 0, mpidr;

        BUG_ON(cpu != smp_processor_id());
        cpuidr = read_cpuid_part_number();
        mpidr = read_cpuid_mpidr();
        for (idx = cpu_compat_table; idx->name; idx++) {
                if (idx->cpuidr == cpuidr)
                        break;
        }
        BUG_ON(!idx || !idx->name);
        setup_cputopo(cpu, idx, mpidr);
}

static void reset_cputopo(void)
{
        struct cpu_compatible *idx;

        memset(glb_cpu_arch, 0, sizeof(struct cpu_arch_info));
        glb_cpu_arch->arch_type = ARCH_UNKNOWN;

        memset(&compat_cputopo, 0, sizeof(compat_cputopo));

        spin_lock(&__cpu_cluster_lock);
        for (idx = cpu_compat_table; idx->name; idx++) {
                struct cpu_cluster *curr, *next;

                if (idx->clscnt == 0)
                        continue;
                BUG_ON(!idx->cluster);

                curr = idx->cluster;
                next = (struct cpu_cluster *)curr->next;
                kfree(curr);

                while (next) {
                        curr = next;
                        next = (struct cpu_cluster *)curr->next;
                        kfree(curr);
                }
                idx->cluster = NULL;
                idx->clscnt = 0;
        }
        spin_unlock(&__cpu_cluster_lock);
}

/* verify cpu topology correctness by device tree.
 * This function is called when current CPU is cpuid!
 */
static void verify_cputopo(const unsigned int cpuid, const u32 mpidr)
{
        struct cputopo_arm *cpuid_topo = &cpu_topology[cpuid];
        struct cpu_compatible *cpt;
        struct cpu_cluster *cls;

        if (!glb_cpu_arch->arch_ready) {
                int i;

                setup_cputopo_def(cpuid);
                for (i = 0; i < nr_cpu_ids; i++)
                        if (!compat_cputopo[i])
                                break;
                if (i == nr_cpu_ids)
                        glb_cpu_arch->arch_ready = true;

                return;
        }

        cpt = compat_cputopo[cpuid];
        BUG_ON(!cpt);
        cls = cpt->cluster;
        while (cls) {
                if (cpu_isset(cpuid, cls->siblings))
                        break;
                cls = cls->next;
        }
        BUG_ON(!cls);
        WARN(cls->cluster_id != cpuid_topo->socket_id,
                 "[%s] cpu id: %d, cluster id (%d) != socket id (%d)\n",
                 __func__, cpuid, cls->cluster_id, cpuid_topo->socket_id);
}

/*
 * return 1 while every cpu is recognizible
 */
void arch_build_cpu_topology_domain(void)
{
        struct device_node *cn = NULL;
        unsigned int cpu = 0;
        u32 mpidr;

        memset(&compat_cputopo, 0, sizeof(compat_cputopo));
        // default by device tree parsing
        while ((cn = of_find_node_by_type(cn, "cpu"))) {
                struct cpu_compatible *idx;
                const u32 *reg;
                int len;

                if (unlikely(cpu >= nr_cpu_ids)) {
                        pr_err("[CPUTOPO][%s] device tree cpu%d is over possible's\n",
                               __func__, cpu);
                        break;
                }

                for (idx = cpu_compat_table; idx->name; idx++)
                        if (of_device_is_compatible(cn, idx->name))
                                break;

                if (!idx || !idx->name) {
                        int cplen;
                        const char *cp;
                        cp = (char *) of_get_property(cn, "compatible", &cplen);
                        pr_err("[CPUTOPO][%s] device tree cpu%d (%s) is not compatible!!\n",
                               __func__, cpu, cp);
                        break;
                }

                reg = of_get_property(cn, "reg", &len);
                if (!reg || len != 4) {
                        pr_err("[CPUTOPO][%s] missing reg property\n", cn->full_name);
                        break;
                }
                mpidr = be32_to_cpup(reg);
                setup_cputopo(cpu, idx, mpidr);
                cpu++;
        }
        glb_cpu_arch->arch_ready = (cpu == nr_cpu_ids);

        if (!glb_cpu_arch->arch_ready) {
                pr_warn("[CPUTOPO][%s] build cpu topology failed, to be handled by mpidr/cpuidr regs!\n", __func__);
                reset_cputopo();
                setup_cputopo_def(smp_processor_id());
        }
}

int arch_cpu_is_big(unsigned int cpu)
{
        int type;

        if (unlikely(cpu >= nr_cpu_ids))
                BUG();

        type = __arch_type();
        switch(type) {
        case ARCH_BIG_LITTLE:
                return (compat_cputopo[cpu] == glb_cpu_arch->compat_big);
        default:
                /* treat as little */
                return 0;
        }
}

int arch_cpu_is_little(unsigned int cpu)
{
        int type;

        if (unlikely(cpu >= nr_cpu_ids))
                BUG();

        type = __arch_type();
        switch(type) {
        case ARCH_BIG_LITTLE:
                return (compat_cputopo[cpu] == glb_cpu_arch->compat_ltt);
        default:
                /* treat as little */
                return 1;
        }
}

int arch_is_multi_cluster(void)
{
        return (__arch_type() == ARCH_MULTI_CLUSTER || __arch_type() == ARCH_BIG_LITTLE);
}

int arch_is_big_little(void)
{
        return (__arch_type() == ARCH_BIG_LITTLE);
}

int arch_get_nr_clusters(void)
{
        return glb_cpu_arch->nr_clusters;
}

int arch_get_cluster_id(unsigned int cpu)
{
        struct cputopo_arm *arm_cputopo = &cpu_topology[cpu];
        struct cpu_compatible *cpt;
        struct cpu_cluster *cls;

        BUG_ON(cpu >= nr_cpu_ids);
        if (!glb_cpu_arch->arch_ready) {
                WARN_ONCE(!glb_cpu_arch->arch_ready, "[CPUTOPO][%s] cpu(%d), socket_id(%d) topology is not ready!\n",
                                  __func__, cpu, arm_cputopo->socket_id);
                if (unlikely(arm_cputopo->socket_id < 0))
                        return 0;
                return arm_cputopo->socket_id;
        }

        cpt = compat_cputopo[cpu];
        BUG_ON(!cpt);
        cls = cpt->cluster;
        while (cls) {
                if (cpu_isset(cpu, cls->siblings))
                        break;
                cls = cls->next;
        }
        BUG_ON(!cls);
        WARN_ONCE(cls->cluster_id != arm_cputopo->socket_id, "[CPUTOPO][%s] cpu(%d): cluster_id(%d) != socket_id(%d) !\n",
                          __func__, cpu, cls->cluster_id, arm_cputopo->socket_id);

        return cls->cluster_id;
}

static struct cpu_cluster *__get_cluster_slowpath(int cluster_id)
{
        int i = 0;
        struct cpu_compatible *cpt;
        struct cpu_cluster *cls;

        for (i = 0; i < nr_cpu_ids; i++) {
                cpt = compat_cputopo[i];
                BUG_ON(!cpt);
                cls = cpt->cluster;
                while (cls) {
                        if (cls->cluster_id == cluster_id)
                                return cls;
                        cls = cls->next;
                }
        }
        return NULL;
}

void arch_get_cluster_cpus(struct cpumask *cpus, int cluster_id)
{
        struct cpu_cluster *cls = NULL;

        cpumask_clear(cpus);

        if (likely(glb_cpu_arch->compat_ltt)) {
                cls = glb_cpu_arch->compat_ltt->cluster;
                while (cls) {
                        if (cls->cluster_id == cluster_id)
                                goto found;
                        cls = cls->next;
                }
        }
        if (likely(glb_cpu_arch->compat_big)) {
                cls = glb_cpu_arch->compat_big->cluster;
                while (cls) {
                        if (cls->cluster_id == cluster_id)
                                goto found;
                        cls = cls->next;
                }
        }

        cls = __get_cluster_slowpath(cluster_id);
        BUG_ON(!cls); // debug only.. remove later...
        if (!cls)
                return;

found:
        cpumask_copy(cpus, &cls->siblings);
}

/*
 * arch_get_big_little_cpus - get big/LITTLE cores in cpumask
 * @big: the cpumask pointer of big cores
 * @little: the cpumask pointer of little cores
 *
 * Treat it as little cores, if it's not big.LITTLE architecture
 */
void arch_get_big_little_cpus(struct cpumask *big, struct cpumask *little)
{
        int type;
        struct cpu_cluster *cls = NULL;
        struct cpumask tmpmask;
        unsigned int cpu;

        if (unlikely(!glb_cpu_arch->arch_ready))
                BUG();

        type = __arch_type();
        spin_lock(&__cpu_cluster_lock);
        switch(type) {
        case ARCH_BIG_LITTLE:
                if (likely(1 == glb_cpu_arch->compat_big->clscnt)) {
                        cls = glb_cpu_arch->compat_big->cluster;
                        cpumask_copy(big, &cls->siblings);
                } else {
                        cls = glb_cpu_arch->compat_big->cluster;
                        while (cls) {
                                cpumask_or(&tmpmask, big, &cls->siblings);
                                cpumask_copy(big, &tmpmask);
                                cls = cls->next;
                        }
                }
                if (likely(1 == glb_cpu_arch->compat_ltt->clscnt)) {
                        cls = glb_cpu_arch->compat_ltt->cluster;
                        cpumask_copy(little, &cls->siblings);
                } else {
                        cls = glb_cpu_arch->compat_ltt->cluster;
                        while (cls) {
                                cpumask_or(&tmpmask, little, &cls->siblings);
                                cpumask_copy(little, &tmpmask);
                                cls = cls->next;
                        }
                }
                break;
        default:
                /* treat as little */
                cpumask_clear(big);
                cpumask_clear(little);
                for_each_possible_cpu(cpu)
                        cpumask_set_cpu(cpu, little);
        }
        spin_unlock(&__cpu_cluster_lock);
}
#else /* !CONFIG_MTK_CPU_TOPOLOGY */
int arch_cpu_is_big(unsigned int cpu) { return 0; }
int arch_cpu_is_little(unsigned int cpu) { return 1; }
int arch_is_big_little(void) { return 0; }

int arch_get_nr_clusters(void)
{
        int max_id = 0;
        unsigned int cpu;

        // assume socket id is monotonic increasing without gap.
        for_each_possible_cpu(cpu) {
                struct cputopo_arm *arm_cputopo = &cpu_topology[cpu];
                if (arm_cputopo->socket_id > max_id)
                        max_id = arm_cputopo->socket_id;
        }
        return max_id+1;
}

int arch_is_multi_cluster(void)
{
        return (arch_get_nr_clusters() > 1 ? 1 : 0);
}

int arch_get_cluster_id(unsigned int cpu)
{
        struct cputopo_arm *arm_cputopo = &cpu_topology[cpu];
        return arm_cputopo->socket_id < 0 ? 0 : arm_cputopo->socket_id;
}

void arch_get_cluster_cpus(struct cpumask *cpus, int cluster_id)
{
        unsigned int cpu, found_id = -1;

        for_each_possible_cpu(cpu) {
                struct cputopo_arm *arm_cputopo = &cpu_topology[cpu];
                if (arm_cputopo->socket_id == cluster_id) {
                        found_id = cluster_id;
                        break;
                }
        }
        if (-1 == found_id || cluster_to_logical_mask(found_id, cpus)) {
                cpumask_clear(cpus);
                for_each_possible_cpu(cpu)
                        cpumask_set_cpu(cpu, cpus);
        }
}
void arch_get_big_little_cpus(struct cpumask *big, struct cpumask *little)
{
    unsigned int cpu;
    cpumask_clear(big);
    cpumask_clear(little);
    for_each_possible_cpu(cpu)
        cpumask_set_cpu(cpu, little);
}
#endif /* CONFIG_MTK_CPU_TOPOLOGY */

/*
 * store_cpu_topology is called at boot when only one cpu is running
 * and with the mutex cpu_hotplug.lock locked, when several cpus have booted,
 * which prevents simultaneous write access to cpu_topology array
 */
void store_cpu_topology(unsigned int cpuid)
{
        struct cputopo_arm *cpuid_topo = &cpu_topology[cpuid];
        unsigned int mpidr;

        /* If the cpu topology has been already set, just return */
        if (cpuid_topo->core_id != -1)
                return;

        mpidr = read_cpuid_mpidr();

        /* create cpu topology mapping */
        if ((mpidr & MPIDR_SMP_BITMASK) == MPIDR_SMP_VALUE) {
                /*
                 * This is a multiprocessor system
                 * multiprocessor format & multiprocessor mode field are set
                 */

                if (mpidr & MPIDR_MT_BITMASK) {
                        /* core performance interdependency */
                        cpuid_topo->thread_id = MPIDR_AFFINITY_LEVEL(mpidr, 0);
                        cpuid_topo->core_id = MPIDR_AFFINITY_LEVEL(mpidr, 1);
                        cpuid_topo->socket_id = MPIDR_AFFINITY_LEVEL(mpidr, 2);
                } else {
                        /* largely independent cores */
                        cpuid_topo->thread_id = -1;
                        cpuid_topo->core_id = MPIDR_AFFINITY_LEVEL(mpidr, 0);
                        cpuid_topo->socket_id = MPIDR_AFFINITY_LEVEL(mpidr, 1);
                }
        } else {
                /*
                 * This is an uniprocessor system
                 * we are in multiprocessor format but uniprocessor system
                 * or in the old uniprocessor format
                 */
                cpuid_topo->thread_id = -1;
                cpuid_topo->core_id = 0;
                cpuid_topo->socket_id = -1;
        }

#ifdef CONFIG_MTK_CPU_TOPOLOGY
        verify_cputopo(cpuid, (u32)mpidr);
#endif

        update_siblings_masks(cpuid);

        update_cpu_power(cpuid, mpidr & MPIDR_HWID_BITMASK);

        printk(KERN_INFO "CPU%u: thread %d, cpu %d, socket %d, mpidr %x\n",
                   cpuid, cpu_topology[cpuid].thread_id,
                   cpu_topology[cpuid].core_id,
                   cpu_topology[cpuid].socket_id, mpidr);
}

/*
 * cluster_to_logical_mask - return cpu logical mask of CPUs in a cluster
 * @socket_id:          cluster HW identifier
 * @cluster_mask:       the cpumask location to be initialized, modified by the
 *                      function only if return value == 0
 *
 * Return:
 *
 * 0 on success
 * -EINVAL if cluster_mask is NULL or there is no record matching socket_id
 */
int cluster_to_logical_mask(unsigned int socket_id, cpumask_t *cluster_mask)
{
        int cpu;

        if (!cluster_mask)
                return -EINVAL;

        for_each_online_cpu(cpu)
                if (socket_id == topology_physical_package_id(cpu)) {
                        cpumask_copy(cluster_mask, topology_core_cpumask(cpu));
                        return 0;
                }

        return -EINVAL;
}

#ifdef CONFIG_SCHED_HMP
static const char * const little_cores[] = {
        "arm,cortex-a53",
        "arm,cortex-a7",
        NULL,
};

static bool is_little_cpu(struct device_node *cn)
{
        const char * const *lc;
        for (lc = little_cores; *lc; lc++)
                if (of_device_is_compatible(cn, *lc)) {
                        return true;
                }
        return false;
}

void __init arch_get_fast_and_slow_cpus(struct cpumask *fast,
                                                                                struct cpumask *slow)
{
        struct device_node *cn = NULL;
        int cpu;

        cpumask_clear(fast);
        cpumask_clear(slow);
        
        /*
         * Use the config options if they are given. This helps testing
         * HMP scheduling on systems without a big.LITTLE architecture.
         */
        if (strlen(CONFIG_HMP_FAST_CPU_MASK) && strlen(CONFIG_HMP_SLOW_CPU_MASK)) {
                if (cpulist_parse(CONFIG_HMP_FAST_CPU_MASK, fast))
                        WARN(1, "Failed to parse HMP fast cpu mask!\n");
                if (cpulist_parse(CONFIG_HMP_SLOW_CPU_MASK, slow))
                        WARN(1, "Failed to parse HMP slow cpu mask!\n");
                return;
        }

        /*
         * Else, parse device tree for little cores.
         */
        while ((cn = of_find_node_by_type(cn, "cpu"))) {

                const u32 *mpidr;
                int len;

                mpidr = of_get_property(cn, "reg", &len);
                if (!mpidr || len != 4) {
                        pr_err("* %s missing reg property\n", cn->full_name);
                        continue;
                }

                cpu = get_logical_index(be32_to_cpup(mpidr));
                if (cpu == -EINVAL) {
                        pr_err("couldn't get logical index for mpidr %x\n",
                                   be32_to_cpup(mpidr));
                        break;
                }

                if (is_little_cpu(cn))
                        cpumask_set_cpu(cpu, slow);
                else
                        cpumask_set_cpu(cpu, fast);
        }

        if (!cpumask_empty(fast) && !cpumask_empty(slow))
                return;

        /*
         * We didn't find both big and little cores so let's call all cores
         * fast as this will keep the system running, with all cores being
         * treated equal.
         */
        cpumask_setall(fast);
        cpumask_clear(slow);
}

struct cpumask hmp_fast_cpu_mask;
struct cpumask hmp_slow_cpu_mask;

void __init arch_get_hmp_domains(struct list_head *hmp_domains_list)
{
        struct hmp_domain *domain;

        arch_get_fast_and_slow_cpus(&hmp_fast_cpu_mask, &hmp_slow_cpu_mask);

        /*
         * Initialize hmp_domains
         * Must be ordered with respect to compute capacity.
         * Fastest domain at head of list.
         */
        if(!cpumask_empty(&hmp_slow_cpu_mask)) {
                domain = (struct hmp_domain *)
                        kmalloc(sizeof(struct hmp_domain), GFP_KERNEL);
                cpumask_copy(&domain->possible_cpus, &hmp_slow_cpu_mask);
                cpumask_and(&domain->cpus, cpu_online_mask, &domain->possible_cpus);
                list_add(&domain->hmp_domains, hmp_domains_list);
        }
        domain = (struct hmp_domain *)
                kmalloc(sizeof(struct hmp_domain), GFP_KERNEL);
        cpumask_copy(&domain->possible_cpus, &hmp_fast_cpu_mask);
        cpumask_and(&domain->cpus, cpu_online_mask, &domain->possible_cpus);
        list_add(&domain->hmp_domains, hmp_domains_list);
}
#endif /* CONFIG_SCHED_HMP */

/*
 * init_cpu_topology is called at boot when only one cpu is running
 * which prevent simultaneous write access to cpu_topology array
 */
void __init init_cpu_topology(void)
{
        unsigned int cpu;

        /* init core mask and power*/
        for_each_possible_cpu(cpu) {
                struct cputopo_arm *cpu_topo = &(cpu_topology[cpu]);

                cpu_topo->thread_id = -1;
                cpu_topo->core_id =  -1;
                cpu_topo->socket_id = -1;
                cpumask_clear(&cpu_topo->core_sibling);
                cpumask_clear(&cpu_topo->thread_sibling);

                set_power_scale(cpu, SCHED_POWER_SCALE);
        }
        smp_wmb();

        parse_dt_topology();
}


#ifdef CONFIG_ARCH_SCALE_INVARIANT_CPU_CAPACITY
#include <linux/cpufreq.h>
#define ARCH_SCALE_INVA_CPU_CAP_PERCLS 1

struct cpufreq_extents {
        u32 max;
        u32 flags;
        u32 const_max;
        u32 throttling;
};
/* Flag set when the governor in use only allows one frequency.
 * Disables scaling.
 */
#define CPUPOWER_FREQINVAR_SINGLEFREQ 0x01
static struct cpufreq_extents freq_scale[CONFIG_NR_CPUS];

static unsigned long get_max_cpu_power(void)
{
        unsigned long max_cpu_power = 0;
        int cpu;
        for_each_online_cpu(cpu){
                if( per_cpu(cpu_scale, cpu) > max_cpu_power)
                        max_cpu_power = per_cpu(cpu_scale, cpu);
        }
        return max_cpu_power;
}

int arch_get_cpu_throttling(int cpu)
{
        return freq_scale[cpu].throttling;
}

/* Called when the CPU Frequency is changed.
 * Once for each CPU.
 */
static int cpufreq_callback(struct notifier_block *nb,
                                                        unsigned long val, void *data)
{
        struct cpufreq_freqs *freq = data;
        int cpu = freq->cpu;
        struct cpufreq_extents *extents;
        unsigned int curr_freq;
#ifdef ARCH_SCALE_INVA_CPU_CAP_PERCLS
        int i = 0;
#endif

        if (freq->flags & CPUFREQ_CONST_LOOPS)
                return NOTIFY_OK;

        if (val != CPUFREQ_POSTCHANGE)
                return NOTIFY_OK;

        /* if dynamic load scale is disabled, set the load scale to 1.0 */
        if (!frequency_invariant_power_enabled) {
                per_cpu(invariant_cpu_capacity, cpu) = per_cpu(base_cpu_capacity, cpu);
                return NOTIFY_OK;
        }

        extents = &freq_scale[cpu];
        if (extents->max < extents->const_max) {
                extents->throttling = 1;
        } else {
                extents->throttling = 0;
        }
        /* If our governor was recognised as a single-freq governor,
         * use curr = max to be sure multiplier is 1.0
         */
        if (extents->flags & CPUPOWER_FREQINVAR_SINGLEFREQ)
                curr_freq = extents->max >> CPUPOWER_FREQSCALE_SHIFT;
        else
                curr_freq = freq->new >> CPUPOWER_FREQSCALE_SHIFT;

#ifdef ARCH_SCALE_INVA_CPU_CAP_PERCLS
        for_each_cpu(i, topology_core_cpumask(cpu)) {
                        per_cpu(invariant_cpu_capacity, i) = DIV_ROUND_UP(
                                (curr_freq * per_cpu(prescaled_cpu_capacity, i)), CPUPOWER_FREQSCALE_DEFAULT);
        }
#else
        per_cpu(invariant_cpu_capacity, cpu) = DIV_ROUND_UP(
                (curr_freq * per_cpu(prescaled_cpu_capacity, cpu)), CPUPOWER_FREQSCALE_DEFAULT);
#endif
        return NOTIFY_OK;
}

/* Called when the CPUFreq governor is changed.
 * Only called for the CPUs which are actually changed by the
 * userspace.
 */
static int cpufreq_policy_callback(struct notifier_block *nb,
                                                                   unsigned long event, void *data)
{
        struct cpufreq_policy *policy = data;
        struct cpufreq_extents *extents;
        int cpu, singleFreq = 0, cpu_capacity;
        static const char performance_governor[] = "performance";
        static const char powersave_governor[] = "powersave";
        unsigned long max_cpu_power;
#ifdef ARCH_SCALE_INVA_CPU_CAP_PERCLS
        int i = 0;
#endif

        if (event == CPUFREQ_START)
                return 0;

        if (event != CPUFREQ_INCOMPATIBLE)
                return 0;

        /* CPUFreq governors do not accurately report the range of
         * CPU Frequencies they will choose from.
         * We recognise performance and powersave governors as
         * single-frequency only.
         */
        if (!strncmp(policy->governor->name, performance_governor,
                                 strlen(performance_governor)) ||
                !strncmp(policy->governor->name, powersave_governor,
                                 strlen(powersave_governor)))
                singleFreq = 1;

        max_cpu_power = get_max_cpu_power();
        /* Make sure that all CPUs impacted by this policy are
         * updated since we will only get a notification when the
         * user explicitly changes the policy on a CPU.
         */
        for_each_cpu(cpu, policy->cpus) {
                /* scale cpu_power to max(1024) */
                cpu_capacity = (per_cpu(cpu_scale, cpu) << CPUPOWER_FREQSCALE_SHIFT)
                        / max_cpu_power;
                extents = &freq_scale[cpu];
                extents->max = policy->max >> CPUPOWER_FREQSCALE_SHIFT;
                extents->const_max = policy->cpuinfo.max_freq >> CPUPOWER_FREQSCALE_SHIFT;
                if (!frequency_invariant_power_enabled) {
                        /* when disabled, invariant_cpu_scale = cpu_scale */
                        per_cpu(base_cpu_capacity, cpu) = CPUPOWER_FREQSCALE_DEFAULT;
                        per_cpu(invariant_cpu_capacity, cpu) = CPUPOWER_FREQSCALE_DEFAULT;
                        /* unused when disabled */
                        per_cpu(prescaled_cpu_capacity, cpu) = CPUPOWER_FREQSCALE_DEFAULT;
                } else {
                        if (singleFreq)
                                extents->flags |= CPUPOWER_FREQINVAR_SINGLEFREQ;
                        else
                                extents->flags &= ~CPUPOWER_FREQINVAR_SINGLEFREQ;
                        per_cpu(base_cpu_capacity, cpu) = cpu_capacity;
#ifdef CONFIG_SCHED_HMP_ENHANCEMENT
                        per_cpu(prescaled_cpu_capacity, cpu) =
                                ((cpu_capacity << CPUPOWER_FREQSCALE_SHIFT) / extents->const_max);
#else
                        per_cpu(prescaled_cpu_capacity, cpu) =
                                ((cpu_capacity << CPUPOWER_FREQSCALE_SHIFT) / extents->max);
#endif

#ifdef ARCH_SCALE_INVA_CPU_CAP_PERCLS
                        for_each_cpu(i, topology_core_cpumask(cpu)) {
                                        per_cpu(invariant_cpu_capacity, i) = DIV_ROUND_UP(
                                                ((policy->cur>>CPUPOWER_FREQSCALE_SHIFT) *
                                                 per_cpu(prescaled_cpu_capacity, i)), CPUPOWER_FREQSCALE_DEFAULT);
                        }
#else
                        per_cpu(invariant_cpu_capacity, cpu) = DIV_ROUND_UP(
                                ((policy->cur>>CPUPOWER_FREQSCALE_SHIFT) *
                                 per_cpu(prescaled_cpu_capacity, cpu)), CPUPOWER_FREQSCALE_DEFAULT);
#endif
                }
        }
        return 0;
}

static struct notifier_block cpufreq_notifier = {
        .notifier_call  = cpufreq_callback,
};
static struct notifier_block cpufreq_policy_notifier = {
        .notifier_call  = cpufreq_policy_callback,
};

static int __init register_topology_cpufreq_notifier(void)
{
        int ret;

        /* init safe defaults since there are no policies at registration */
        for (ret = 0; ret < CONFIG_NR_CPUS; ret++) {
                /* safe defaults */
                freq_scale[ret].max = CPUPOWER_FREQSCALE_DEFAULT;
                per_cpu(base_cpu_capacity, ret) = CPUPOWER_FREQSCALE_DEFAULT;
                per_cpu(invariant_cpu_capacity, ret) = CPUPOWER_FREQSCALE_DEFAULT;
                per_cpu(prescaled_cpu_capacity, ret) = CPUPOWER_FREQSCALE_DEFAULT;
        }

        pr_info("topology: registering cpufreq notifiers for scale-invariant CPU Power\n");
        ret = cpufreq_register_notifier(&cpufreq_policy_notifier,
                                                                        CPUFREQ_POLICY_NOTIFIER);

        if (ret != -EINVAL)
                ret = cpufreq_register_notifier(&cpufreq_notifier,
                                                                                CPUFREQ_TRANSITION_NOTIFIER);

        return ret;
}

core_initcall(register_topology_cpufreq_notifier);
#endif /* CONFIG_ARCH_SCALE_INVARIANT_CPU_CAPACITY */