From afa22a350435e3f2f8b6fe988dfd0e6ac16fab11 Mon Sep 17 00:00:00 2001 From: Andrew Chant Date: Tue, 9 Jun 2020 10:39:14 -0700 Subject: [PATCH] rebalance_interrupts: one shot IRQ rebalancer rebalance_interrupts is a one shot IRQ rebalancer. It scans /sys/kernel/irq for related IRQ actions that haven't any affinity to a core. It then assigns them evenly to the policy0 group cores. Bug: 148403062 Test: ran "rebalance_interrupts" as root after boot, saw no additional IRQs rebalanced. Verified via /proc/interrupts that interrupts had been distributed. Change-Id: I76745d4620024a1f21ecf70d11e0c14d2eb85cef --- rebalance_interrupts/Android.bp | 8 + .../rebalance_interrupts-samsung.rc | 12 + rebalance_interrupts/rebalance_interrupts.cpp | 250 ++++++++++++++++++ 3 files changed, 270 insertions(+) create mode 100644 rebalance_interrupts/Android.bp create mode 100644 rebalance_interrupts/rebalance_interrupts-samsung.rc create mode 100644 rebalance_interrupts/rebalance_interrupts.cpp diff --git a/rebalance_interrupts/Android.bp b/rebalance_interrupts/Android.bp new file mode 100644 index 0000000..813cff1 --- /dev/null +++ b/rebalance_interrupts/Android.bp @@ -0,0 +1,8 @@ +cc_binary { + name: "rebalance_interrupts-samsung", + init_rc: ["rebalance_interrupts-samsung.rc"], + srcs: ["rebalance_interrupts.cpp"], + shared_libs: ["libbase", + "liblog",], + proprietary: true, +} diff --git a/rebalance_interrupts/rebalance_interrupts-samsung.rc b/rebalance_interrupts/rebalance_interrupts-samsung.rc new file mode 100644 index 0000000..ef7b53e --- /dev/null +++ b/rebalance_interrupts/rebalance_interrupts-samsung.rc @@ -0,0 +1,12 @@ +service vendor.rebalance_interrupts-samsung /vendor/bin/rebalance_interrupts-samsung + disabled + oneshot + user root + group system + +on early-boot + start vendor.rebalance_interrupts-samsung + +on property:sys.boot_completed=1 + start vendor.rebalance_interrupts-samsung + diff --git a/rebalance_interrupts/rebalance_interrupts.cpp b/rebalance_interrupts/rebalance_interrupts.cpp new file mode 100644 index 0000000..b0d2f69 --- /dev/null +++ b/rebalance_interrupts/rebalance_interrupts.cpp @@ -0,0 +1,250 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * rebalance-interrupts: + * + * One-shot distribution of unassigned* IRQs to CPU cores. + * Useful for devices with the ARM-GIC-v3, as the Linux driver will take any + * interrupt assigned with an all-cores mask and always have it run on core 0. + * + * This should be run once, long enough after boot that all drivers have + * registered their interrupts. + * + * This program is configured to spread the load across all the cores in + * CPUFREQ policy 0. This is because other cores may be hotplugged in + * or out, and if hotplugged out the interrupts would be sent to core0 always. + * + * It might be wise to avoid core0 so that any later-added IRQs don't overcrowd + * core 0. + * + * Any program that has an actual IRQ related performance constraint should + * override any settings assigned by this and assign the IRQ to the same + * core as the code whose performance is impacted by the IRQ. + * + */ + +#include +#include + +#include +#include +#include +#include + +#define LOG_TAG "rebalance_interrupts" + +#include +#include +#include +#include +#include + + +#define POLICY0_CORES_PATH "/sys/devices/system/cpu/cpufreq/policy0/affected_cpus" +#define SYSFS_IRQDIR "/sys/kernel/irq" +#define PROC_IRQDIR "/proc/irq" + +using android::base::ParseInt; +using android::base::ParseUint; +using android::base::ReadFileToString; +using android::base::Trim; +using android::base::WriteStringToFile; +using std::list; +using std::map; +using std::pair; +using std::string; +using std::vector; + +// Return a vector of strings describing the affected CPUs for cpufreq +// Policy 0. +vector Policy0AffectedCpus() { + string policy0_cores_unparsed; + if (!ReadFileToString(POLICY0_CORES_PATH, &policy0_cores_unparsed)) + return vector(); + string policy0_trimmed = android::base::Trim(policy0_cores_unparsed); + vector cpus_as_string = android::base::Split(policy0_trimmed, " "); + + vector cpus_as_int; + for (int i = 0; i < cpus_as_string.size(); ++i) { + int cpu; + if (!ParseInt(cpus_as_string[i].c_str(), &cpu)) + return vector(); + cpus_as_int.push_back(cpu); + } + return cpus_as_int; +} + +// Return a vector of strings describing the CPU masks for cpufreq Policy 0. +vector Policy0CpuMasks() { + vector cpus = Policy0AffectedCpus(); + vector cpu_masks; + for (int i = 0; i < cpus.size(); ++i) + cpu_masks.push_back(fmt::format("{0:02x}", 1 << cpus[i])); + return cpu_masks; +} + +// Read the actions for the given irq# from sysfs, and add it to action_to_irq +bool AddEntryToIrqmap(const char* irq, + map>& action_to_irqs) { + const string irq_base(SYSFS_IRQDIR "/"); + string irq_actions_path = irq_base + irq + "/actions"; + + string irq_actions; + if (!ReadFileToString(irq_actions_path, &irq_actions)) + return false; + + irq_actions = Trim(irq_actions); + + if (irq_actions == "(null)") + irq_actions = ""; + + action_to_irqs[irq_actions].push_back(irq); + + return true; +} + +// Get a mapping of driver "action" to IRQ#s for each IRQ# in +// SYSFS_IRQDIR. +bool GetIrqmap(map>& action_to_irqs) { + bool some_success = false; + std::unique_ptr irq_dir(opendir(SYSFS_IRQDIR), closedir); + if (!irq_dir) { + PLOG(ERROR) << "opening dir " SYSFS_IRQDIR; + return false; + } + + struct dirent* entry; + while ((entry = readdir(irq_dir.get()))) { + + // If the directory entry isn't a parsable number, skip it. + // . and .. get skipped here. + unsigned throwaway; + if (!ParseUint(entry->d_name, &throwaway)) + continue; + + some_success |= AddEntryToIrqmap(entry->d_name, action_to_irqs); + } + return some_success; +} + +// Given a map of irq actions -> IRQs, +// find out which ones haven't been assigned and add those to +// rebalance_actions. +void FindUnassignedIrqs(const map>& action_to_irqs, + list>>& rebalance_actions) { + for (const auto &action_to_irqs_entry: action_to_irqs) { + bool rebalance = true; + for (const auto& irq: action_to_irqs_entry.second) { + string smp_affinity; + string proc_path(PROC_IRQDIR "/"); + proc_path += irq + "/smp_affinity"; + ReadFileToString(proc_path, &smp_affinity); + smp_affinity = Trim(smp_affinity); + + // Try to respect previoulsy set IRQ affinities. + // On ARM interrupt controllers under Linux, if an IRQ is assigned + // to more than one core it will only be assigned to the lowest core. + // Assume any IRQ which is set to more than one core in the lowest four + // CPUs hasn't been assigned and needs to be rebalanced. + if (smp_affinity.back() == '0' || + smp_affinity.back() == '1' || + smp_affinity.back() == '2' || + smp_affinity.back() == '4' || + smp_affinity.back() == '8') { + rebalance = false; + } + + // Treat each unnamed action IRQ as independent. + if (action_to_irqs_entry.first.empty()) { + if (rebalance) { + pair> empty_action_irq; + empty_action_irq.first = ""; + empty_action_irq.second.push_back(irq); + rebalance_actions.push_back(empty_action_irq); + } + rebalance = true; + } + } + if (rebalance && !action_to_irqs_entry.first.empty()) { + rebalance_actions.push_back(std::make_pair(action_to_irqs_entry.first, + action_to_irqs_entry.second)); + } + } +} + +// Read the file at `path`, Trim whitespace, see if it matches `expected_value`. +// Print the results to stdout. +void ReportIfAffinityUpdated(const std::string expected_value, + const std::string path) { + std::string readback, report; + ReadFileToString(path, &readback); + readback = Trim(readback); + if (readback != expected_value) { + report += "Unable to set "; + } else { + report += "Success setting "; + } + report += path; + report += ": found " + readback + " vs " + expected_value + "\n"; + LOG(DEBUG) << report; +} + +// Evenly distribute the IRQ actions across all the Policy0 CPUs. +// Assign all the IRQs of an action to a single CPU core. +bool RebalanceIrqs(const list>>& action_to_irqs) { + int mask_index = 0; + std::vector affinity_masks = Policy0CpuMasks(); + + if (affinity_masks.empty()) { + LOG(ERROR) << "Unable to find Policy0 CPUs for IRQ assignment."; + return false; + } + + for (const auto &action_to_irq: action_to_irqs) { + for (const auto& irq: action_to_irq.second) { + std::string affinity_path(PROC_IRQDIR "/"); + affinity_path += irq + "/smp_affinity"; + WriteStringToFile(affinity_masks[mask_index], affinity_path); + ReportIfAffinityUpdated(affinity_masks[mask_index], affinity_path); + } + mask_index = (mask_index + 1) % affinity_masks.size(); + } + return true; +} + +int main(int /* argc */, char* /* argv */[]) { + map> irq_mapping; + list>> action_to_irqs; + + // Find the mapping of "irq actions" to IRQs. + // Each IRQ has an assocatied irq_actions field, showing the actions + // associated with it. Multiple IRQs have the same actions. + // Generate the mapping of actions to IRQs with that action, + // as these IRQs should all be mapped to the same cores. + if (!GetIrqmap(irq_mapping)) { + LOG(ERROR) << "Unable to read IRQ mappings. Are you root?"; + return 1; + } + + // Some IRQs are already assigned to a subset of cores, usually for + // good reason (like some drivers have an IRQ per core, for per-core + // queues.) Find the set of IRQs that haven't been mapped to specific + // cores. + FindUnassignedIrqs(irq_mapping, action_to_irqs); + + // Distribute the rebalancable IRQs across all cores. + return RebalanceIrqs(action_to_irqs) ? 0 : 1; +} + -- 2.20.1