// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2010-2021, The Linux Foundation. All rights reserved. * Copyright (c) 2022-2024, Qualcomm Innovation Center, Inc. All rights reserved. */ #include #include #include #include #include #include #include #include #include #include #include "kgsl_device.h" #include "kgsl_bus.h" #include "kgsl_pwrscale.h" #include "kgsl_sysfs.h" #include "kgsl_trace.h" #include "kgsl_util.h" #define UPDATE_BUSY_VAL 1000000 #define KGSL_MAX_BUSLEVELS 20 /* Order deeply matters here because reasons. New entries go on the end */ static const char * const clocks[KGSL_MAX_CLKS] = { "src_clk", "core_clk", "iface_clk", "mem_clk", "mem_iface_clk", "alt_mem_iface_clk", "rbbmtimer_clk", "gtcu_clk", "gtbu_clk", "gtcu_iface_clk", "alwayson_clk", "isense_clk", "rbcpr_clk", "iref_clk", "gmu_clk", "ahb_clk", "smmu_vote", "apb_pclk", }; static void kgsl_pwrctrl_clk(struct kgsl_device *device, bool state, int requested_state); static int kgsl_pwrctrl_pwrrail(struct kgsl_device *device, bool state); static int _isense_clk_set_rate(struct kgsl_pwrctrl *pwr, int level); static int kgsl_pwrctrl_clk_set_rate(struct clk *grp_clk, unsigned int freq, const char *name); static void _gpu_clk_prepare_enable(struct kgsl_device *device, struct clk *clk, const char *name); static void _bimc_clk_prepare_enable(struct kgsl_device *device, struct clk *clk, const char *name); /** * _adjust_pwrlevel() - Given a requested power level do bounds checking on the * constraints and return the nearest possible level * @device: Pointer to the kgsl_device struct * @level: Requested level * @pwrc: Pointer to the power constraint to be applied * * Apply thermal and max/min limits first. Then force the level with a * constraint if one exists. */ static unsigned int _adjust_pwrlevel(struct kgsl_pwrctrl *pwr, int level, struct kgsl_pwr_constraint *pwrc) { unsigned int max_pwrlevel = max_t(unsigned int, pwr->thermal_pwrlevel, pwr->max_pwrlevel); unsigned int min_pwrlevel = min_t(unsigned int, pwr->thermal_pwrlevel_floor, pwr->min_pwrlevel); /* Ensure that max/min pwrlevels are within thermal max/min limits */ max_pwrlevel = min_t(unsigned int, max_pwrlevel, pwr->thermal_pwrlevel_floor); min_pwrlevel = max_t(unsigned int, min_pwrlevel, pwr->thermal_pwrlevel); switch (pwrc->type) { case KGSL_CONSTRAINT_PWRLEVEL: { switch (pwrc->sub_type) { case KGSL_CONSTRAINT_PWR_MAX: return max_pwrlevel; case KGSL_CONSTRAINT_PWR_MIN: return min_pwrlevel; default: break; } } break; } if (level < max_pwrlevel) return max_pwrlevel; if (level > min_pwrlevel) return min_pwrlevel; return level; } /** * kgsl_pwrctrl_pwrlevel_change_settings() - Program h/w during powerlevel * transitions * @device: Pointer to the kgsl_device struct * @post: flag to check if the call is before/after the clk_rate change * @wake_up: flag to check if device is active or waking up */ static void kgsl_pwrctrl_pwrlevel_change_settings(struct kgsl_device *device, bool post) { struct kgsl_pwrctrl *pwr = &device->pwrctrl; unsigned int old = pwr->previous_pwrlevel; unsigned int new = pwr->active_pwrlevel; if (device->state != KGSL_STATE_ACTIVE) return; if (old == new) return; device->ftbl->pwrlevel_change_settings(device, old, new, post); } /** * kgsl_pwrctrl_adjust_pwrlevel() - Adjust the power level if * required by thermal, max/min, constraints, etc * @device: Pointer to the kgsl_device struct * @new_level: Requested powerlevel, an index into the pwrlevel array */ unsigned int kgsl_pwrctrl_adjust_pwrlevel(struct kgsl_device *device, unsigned int new_level) { struct kgsl_pwrctrl *pwr = &device->pwrctrl; unsigned int old_level = pwr->active_pwrlevel; bool reset = false; /* If a pwr constraint is expired, remove it */ if ((pwr->constraint.type != KGSL_CONSTRAINT_NONE) && (time_after(jiffies, pwr->constraint.expires))) { struct kgsl_context *context = kgsl_context_get(device, pwr->constraint.owner_id); /* We couldn't get a reference, clear the constraint */ if (!context) { reset = true; goto done; } /* * If the last timestamp that set the constraint has retired, * clear the constraint */ if (kgsl_check_timestamp(device, context, pwr->constraint.owner_timestamp)) { reset = true; kgsl_context_put(context); goto done; } /* * Increase the timeout to keep the constraint at least till * the timestamp retires */ pwr->constraint.expires = jiffies + msecs_to_jiffies(device->pwrctrl.interval_timeout); kgsl_context_put(context); } done: if (reset) { /* Trace the constraint being un-set by the driver */ trace_kgsl_constraint(device, pwr->constraint.type, old_level, 0); /*Invalidate the constraint set */ pwr->constraint.expires = 0; pwr->constraint.type = KGSL_CONSTRAINT_NONE; } /* * Adjust the power level if required by thermal, max/min, * constraints, etc */ return _adjust_pwrlevel(pwr, new_level, &pwr->constraint); } /** * kgsl_pwrctrl_pwrlevel_change() - Validate and change power levels * @device: Pointer to the kgsl_device struct * @new_level: Requested powerlevel, an index into the pwrlevel array * * Check that any power level constraints are still valid. Update the * requested level according to any thermal, max/min, or power constraints. * If a new GPU level is going to be set, update the bus to that level's * default value. Do not change the bus if a constraint keeps the new * level at the current level. Set the new GPU frequency. */ void kgsl_pwrctrl_pwrlevel_change(struct kgsl_device *device, unsigned int new_level) { struct kgsl_pwrctrl *pwr = &device->pwrctrl; struct kgsl_pwrlevel *pwrlevel; unsigned int old_level = pwr->active_pwrlevel; new_level = kgsl_pwrctrl_adjust_pwrlevel(device, new_level); if (new_level == old_level) return; kgsl_pwrscale_update_stats(device); /* * Set the active and previous powerlevel first in case the clocks are * off - if we don't do this then the pwrlevel change won't take effect * when the clocks come back */ pwr->active_pwrlevel = new_level; pwr->previous_pwrlevel = old_level; /* * If the bus is running faster than its default level and the GPU * frequency is moving down keep the DDR at a relatively high level. */ if (pwr->bus_mod < 0 || new_level < old_level) { pwr->bus_mod = 0; pwr->bus_percent_ab = 0; } /* * Update the bus before the GPU clock to prevent underrun during * frequency increases. */ if (new_level < old_level) kgsl_bus_update(device, KGSL_BUS_VOTE_ON); pwrlevel = &pwr->pwrlevels[pwr->active_pwrlevel]; /* Change register settings if any BEFORE pwrlevel change*/ kgsl_pwrctrl_pwrlevel_change_settings(device, 0); device->ftbl->gpu_clock_set(device, pwr->active_pwrlevel); _isense_clk_set_rate(pwr, pwr->active_pwrlevel); trace_kgsl_pwrlevel(device, pwr->active_pwrlevel, pwrlevel->gpu_freq, pwr->previous_pwrlevel, pwr->pwrlevels[old_level].gpu_freq); trace_gpu_frequency(pwrlevel->gpu_freq/1000, 0); /* Update the bus after GPU clock decreases. */ if (new_level > old_level) kgsl_bus_update(device, KGSL_BUS_VOTE_ON); /* * Some targets do not support the bandwidth requirement of * GPU at TURBO, for such targets we need to set GPU-BIMC * interface clocks to TURBO directly whenever GPU runs at * TURBO. The TURBO frequency of gfx-bimc need to be defined * in target device tree. */ if (pwr->gpu_bimc_int_clk) { if (pwr->active_pwrlevel == 0 && !pwr->gpu_bimc_interface_enabled) { kgsl_pwrctrl_clk_set_rate(pwr->gpu_bimc_int_clk, pwr->gpu_bimc_int_clk_freq, "bimc_gpu_clk"); _bimc_clk_prepare_enable(device, pwr->gpu_bimc_int_clk, "bimc_gpu_clk"); pwr->gpu_bimc_interface_enabled = true; } else if (pwr->previous_pwrlevel == 0 && pwr->gpu_bimc_interface_enabled) { clk_disable_unprepare(pwr->gpu_bimc_int_clk); pwr->gpu_bimc_interface_enabled = false; } } /* Change register settings if any AFTER pwrlevel change*/ kgsl_pwrctrl_pwrlevel_change_settings(device, 1); } void kgsl_pwrctrl_set_constraint(struct kgsl_device *device, struct kgsl_pwr_constraint *pwrc, uint32_t id, u32 ts) { unsigned int constraint; struct kgsl_pwr_constraint *pwrc_old; if (device == NULL || pwrc == NULL) return; constraint = _adjust_pwrlevel(&device->pwrctrl, device->pwrctrl.active_pwrlevel, pwrc); pwrc_old = &device->pwrctrl.constraint; /* * If a constraint is already set, set a new constraint only * if it is faster. If the requested constraint is the same * as the current one, update ownership and timestamp. */ if ((pwrc_old->type == KGSL_CONSTRAINT_NONE) || (pwrc_old->sub_type == KGSL_CONSTRAINT_PWR_MIN && pwrc->sub_type == KGSL_CONSTRAINT_PWR_MAX)) { pwrc_old->type = pwrc->type; pwrc_old->sub_type = pwrc->sub_type; pwrc_old->owner_id = id; pwrc_old->expires = jiffies + msecs_to_jiffies(device->pwrctrl.interval_timeout); pwrc_old->owner_timestamp = ts; kgsl_pwrctrl_pwrlevel_change(device, constraint); /* Trace the constraint being set by the driver */ trace_kgsl_constraint(device, pwrc_old->type, constraint, 1); } else if ((pwrc_old->type == pwrc->type) && (pwrc_old->sub_type == pwrc->sub_type)) { pwrc_old->owner_id = id; pwrc_old->owner_timestamp = ts; pwrc_old->expires = jiffies + msecs_to_jiffies(device->pwrctrl.interval_timeout); } } static int kgsl_pwrctrl_set_thermal_limit(struct kgsl_device *device, u32 level) { struct kgsl_pwrctrl *pwr = &device->pwrctrl; int ret = -EINVAL; if (level >= pwr->num_pwrlevels) level = pwr->num_pwrlevels - 1; if (dev_pm_qos_request_active(&pwr->sysfs_thermal_req)) ret = dev_pm_qos_update_request(&pwr->sysfs_thermal_req, (pwr->pwrlevels[level].gpu_freq / 1000)); return (ret < 0) ? ret : 0; } static ssize_t thermal_pwrlevel_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct kgsl_device *device = dev_get_drvdata(dev); int ret; u32 level; ret = kstrtou32(buf, 0, &level); if (ret) return ret; ret = kgsl_pwrctrl_set_thermal_limit(device, level); if (ret) return ret; return count; } static ssize_t thermal_pwrlevel_show(struct device *dev, struct device_attribute *attr, char *buf) { struct kgsl_device *device = dev_get_drvdata(dev); struct kgsl_pwrctrl *pwr = &device->pwrctrl; return scnprintf(buf, PAGE_SIZE, "%d\n", pwr->thermal_pwrlevel); } static ssize_t max_pwrlevel_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct kgsl_device *device = dev_get_drvdata(dev); struct kgsl_pwrctrl *pwr = &device->pwrctrl; int ret; unsigned int level = 0; ret = kstrtou32(buf, 0, &level); if (ret) return ret; mutex_lock(&device->mutex); /* You can't set a maximum power level lower than the minimum */ if (level > pwr->min_pwrlevel) level = pwr->min_pwrlevel; pwr->max_pwrlevel = level; /* Update the current level using the new limit */ kgsl_pwrctrl_pwrlevel_change(device, pwr->active_pwrlevel); mutex_unlock(&device->mutex); return count; } static ssize_t max_pwrlevel_show(struct device *dev, struct device_attribute *attr, char *buf) { struct kgsl_device *device = dev_get_drvdata(dev); struct kgsl_pwrctrl *pwr = &device->pwrctrl; return scnprintf(buf, PAGE_SIZE, "%u\n", pwr->max_pwrlevel); } static void kgsl_pwrctrl_min_pwrlevel_set(struct kgsl_device *device, int level) { struct kgsl_pwrctrl *pwr = &device->pwrctrl; mutex_lock(&device->mutex); if (level > pwr->min_render_pwrlevel) level = pwr->min_render_pwrlevel; /* You can't set a minimum power level lower than the maximum */ if (level < pwr->max_pwrlevel) level = pwr->max_pwrlevel; pwr->min_pwrlevel = level; /* Update the current level using the new limit */ kgsl_pwrctrl_pwrlevel_change(device, pwr->active_pwrlevel); mutex_unlock(&device->mutex); } static ssize_t min_pwrlevel_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct kgsl_device *device = dev_get_drvdata(dev); int ret; unsigned int level = 0; ret = kstrtou32(buf, 0, &level); if (ret) return ret; kgsl_pwrctrl_min_pwrlevel_set(device, level); return count; } static ssize_t min_pwrlevel_show(struct device *dev, struct device_attribute *attr, char *buf) { struct kgsl_device *device = dev_get_drvdata(dev); struct kgsl_pwrctrl *pwr = &device->pwrctrl; return scnprintf(buf, PAGE_SIZE, "%u\n", pwr->min_pwrlevel); } static ssize_t num_pwrlevels_show(struct device *dev, struct device_attribute *attr, char *buf) { struct kgsl_device *device = dev_get_drvdata(dev); struct kgsl_pwrctrl *pwr = &device->pwrctrl; return scnprintf(buf, PAGE_SIZE, "%d\n", pwr->num_pwrlevels); } /* Given a GPU clock value, return the lowest matching powerlevel */ static int _get_nearest_pwrlevel(struct kgsl_pwrctrl *pwr, unsigned int clock) { int i; for (i = pwr->num_pwrlevels - 1; i >= 0; i--) { if (abs(pwr->pwrlevels[i].gpu_freq - clock) < 5000000) return i; } return -ERANGE; } static ssize_t max_gpuclk_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct kgsl_device *device = dev_get_drvdata(dev); u32 freq; int ret, level; ret = kstrtou32(buf, 0, &freq); if (ret) return ret; level = _get_nearest_pwrlevel(&device->pwrctrl, freq); if (level < 0) return level; /* * You would think this would set max_pwrlevel but the legacy behavior * is that it set thermal_pwrlevel instead so we don't want to mess with * that. */ ret = kgsl_pwrctrl_set_thermal_limit(device, level); if (ret) return ret; return count; } static ssize_t max_gpuclk_show(struct device *dev, struct device_attribute *attr, char *buf) { struct kgsl_device *device = dev_get_drvdata(dev); struct kgsl_pwrctrl *pwr = &device->pwrctrl; return scnprintf(buf, PAGE_SIZE, "%d\n", device->pwrctrl.pwrlevels[pwr->thermal_pwrlevel].gpu_freq); } static ssize_t gpuclk_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct kgsl_device *device = dev_get_drvdata(dev); struct kgsl_pwrctrl *pwr = &device->pwrctrl; unsigned int val = 0; int ret, level; ret = kstrtou32(buf, 0, &val); if (ret) return ret; mutex_lock(&device->mutex); level = _get_nearest_pwrlevel(pwr, val); if (level >= 0) kgsl_pwrctrl_pwrlevel_change(device, (unsigned int) level); mutex_unlock(&device->mutex); return count; } static ssize_t gpuclk_show(struct device *dev, struct device_attribute *attr, char *buf) { struct kgsl_device *device = dev_get_drvdata(dev); return scnprintf(buf, PAGE_SIZE, "%ld\n", kgsl_pwrctrl_active_freq(&device->pwrctrl)); } static ssize_t idle_timer_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { unsigned int val = 0; struct kgsl_device *device = dev_get_drvdata(dev); int ret; ret = kstrtou32(buf, 0, &val); if (ret) return ret; /* * We don't quite accept a maximum of 0xFFFFFFFF due to internal jiffy * math, so make sure the value falls within the largest offset we can * deal with */ if (val > jiffies_to_usecs(MAX_JIFFY_OFFSET)) return -EINVAL; mutex_lock(&device->mutex); device->pwrctrl.interval_timeout = val; mutex_unlock(&device->mutex); return count; } static ssize_t idle_timer_show(struct device *dev, struct device_attribute *attr, char *buf) { struct kgsl_device *device = dev_get_drvdata(dev); return scnprintf(buf, PAGE_SIZE, "%u\n", device->pwrctrl.interval_timeout); } static ssize_t minbw_timer_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { /* minbw_timer is deprecated, so return EOPNOTSUPP */ return -EOPNOTSUPP; } static ssize_t minbw_timer_show(struct device *dev, struct device_attribute *attr, char *buf) { /* minbw_timer is deprecated, so return it as always disabled */ return scnprintf(buf, PAGE_SIZE, "0\n"); } static ssize_t gpubusy_show(struct device *dev, struct device_attribute *attr, char *buf) { int ret; struct kgsl_device *device = dev_get_drvdata(dev); struct kgsl_clk_stats *stats = &device->pwrctrl.clk_stats; ret = scnprintf(buf, PAGE_SIZE, "%7d %7d\n", stats->busy_old, stats->total_old); if (!test_bit(KGSL_PWRFLAGS_AXI_ON, &device->pwrctrl.power_flags)) { stats->busy_old = 0; stats->total_old = 0; } return ret; } static ssize_t gpu_available_frequencies_show(struct device *dev, struct device_attribute *attr, char *buf) { struct kgsl_device *device = dev_get_drvdata(dev); struct kgsl_pwrctrl *pwr = &device->pwrctrl; int index, num_chars = 0; for (index = 0; index < pwr->num_pwrlevels; index++) { num_chars += scnprintf(buf + num_chars, PAGE_SIZE - num_chars - 1, "%d ", pwr->pwrlevels[index].gpu_freq); /* One space for trailing null and another for the newline */ if (num_chars >= PAGE_SIZE - 2) break; } buf[num_chars++] = '\n'; return num_chars; } static ssize_t gpu_clock_stats_show(struct device *dev, struct device_attribute *attr, char *buf) { struct kgsl_device *device = dev_get_drvdata(dev); struct kgsl_pwrctrl *pwr = &device->pwrctrl; int index, num_chars = 0; mutex_lock(&device->mutex); kgsl_pwrscale_update_stats(device); mutex_unlock(&device->mutex); for (index = 0; index < pwr->num_pwrlevels; index++) num_chars += scnprintf(buf + num_chars, PAGE_SIZE - num_chars, "%llu ", pwr->clock_times[index]); if (num_chars < PAGE_SIZE) buf[num_chars++] = '\n'; return num_chars; } static ssize_t reset_count_show(struct device *dev, struct device_attribute *attr, char *buf) { struct kgsl_device *device = dev_get_drvdata(dev); return scnprintf(buf, PAGE_SIZE, "%d\n", device->reset_counter); } static void __force_on(struct kgsl_device *device, int flag, int on) { if (on) { switch (flag) { case KGSL_PWRFLAGS_CLK_ON: /* make sure pwrrail is ON before enabling clocks */ kgsl_pwrctrl_pwrrail(device, true); kgsl_pwrctrl_clk(device, true, KGSL_STATE_ACTIVE); break; case KGSL_PWRFLAGS_AXI_ON: kgsl_pwrctrl_axi(device, true); break; case KGSL_PWRFLAGS_POWER_ON: kgsl_pwrctrl_pwrrail(device, true); break; } set_bit(flag, &device->pwrctrl.ctrl_flags); } else { clear_bit(flag, &device->pwrctrl.ctrl_flags); } } static ssize_t __force_on_show(struct device *dev, struct device_attribute *attr, char *buf, int flag) { struct kgsl_device *device = dev_get_drvdata(dev); return scnprintf(buf, PAGE_SIZE, "%d\n", test_bit(flag, &device->pwrctrl.ctrl_flags)); } static ssize_t __force_on_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count, int flag) { unsigned int val = 0; struct kgsl_device *device = dev_get_drvdata(dev); int ret; if (gmu_core_gpmu_isenabled(device)) return -EOPNOTSUPP; ret = kstrtou32(buf, 0, &val); if (ret) return ret; mutex_lock(&device->mutex); __force_on(device, flag, val); mutex_unlock(&device->mutex); return count; } static ssize_t force_clk_on_show(struct device *dev, struct device_attribute *attr, char *buf) { return __force_on_show(dev, attr, buf, KGSL_PWRFLAGS_CLK_ON); } static ssize_t force_clk_on_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { return __force_on_store(dev, attr, buf, count, KGSL_PWRFLAGS_CLK_ON); } static ssize_t force_bus_on_show(struct device *dev, struct device_attribute *attr, char *buf) { return __force_on_show(dev, attr, buf, KGSL_PWRFLAGS_AXI_ON); } static ssize_t force_bus_on_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { return __force_on_store(dev, attr, buf, count, KGSL_PWRFLAGS_AXI_ON); } static ssize_t force_rail_on_show(struct device *dev, struct device_attribute *attr, char *buf) { return __force_on_show(dev, attr, buf, KGSL_PWRFLAGS_POWER_ON); } static ssize_t force_rail_on_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { return __force_on_store(dev, attr, buf, count, KGSL_PWRFLAGS_POWER_ON); } static ssize_t force_no_nap_show(struct device *dev, struct device_attribute *attr, char *buf) { /* force_no_nap is deprecated, so return it as always disabled */ return scnprintf(buf, PAGE_SIZE, "0\n"); } static ssize_t force_no_nap_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { /* force_no_nap is deprecated, so return EOPNOTSUPP */ return -EOPNOTSUPP; } static ssize_t bus_split_show(struct device *dev, struct device_attribute *attr, char *buf) { struct kgsl_device *device = dev_get_drvdata(dev); return scnprintf(buf, PAGE_SIZE, "%d\n", device->pwrctrl.bus_control); } static ssize_t bus_split_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { unsigned int val = 0; struct kgsl_device *device = dev_get_drvdata(dev); int ret; ret = kstrtou32(buf, 0, &val); if (ret) return ret; mutex_lock(&device->mutex); device->pwrctrl.bus_control = val ? true : false; mutex_unlock(&device->mutex); return count; } static ssize_t default_pwrlevel_show(struct device *dev, struct device_attribute *attr, char *buf) { struct kgsl_device *device = dev_get_drvdata(dev); return scnprintf(buf, PAGE_SIZE, "%d\n", device->pwrctrl.default_pwrlevel); } static ssize_t default_pwrlevel_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct kgsl_device *device = dev_get_drvdata(dev); struct kgsl_pwrctrl *pwr = &device->pwrctrl; struct kgsl_pwrscale *pwrscale = &device->pwrscale; int ret; unsigned int level = 0; ret = kstrtou32(buf, 0, &level); if (ret) return ret; if (level >= pwr->num_pwrlevels) return count; mutex_lock(&device->mutex); pwr->default_pwrlevel = level; pwrscale->gpu_profile.profile.initial_freq = pwr->pwrlevels[level].gpu_freq; mutex_unlock(&device->mutex); return count; } static ssize_t popp_show(struct device *dev, struct device_attribute *attr, char *buf) { /* POPP is deprecated, so return it as always disabled */ return scnprintf(buf, PAGE_SIZE, "0\n"); } static ssize_t _gpu_busy_show(struct kgsl_device *device, char *buf) { int ret; struct kgsl_clk_stats *stats = &device->pwrctrl.clk_stats; unsigned int busy_percent = 0; if (stats->total_old != 0) busy_percent = (stats->busy_old * 100) / stats->total_old; ret = scnprintf(buf, PAGE_SIZE, "%d %%\n", busy_percent); /* Reset the stats if GPU is OFF */ if (!test_bit(KGSL_PWRFLAGS_AXI_ON, &device->pwrctrl.power_flags)) { stats->busy_old = 0; stats->total_old = 0; } return ret; } static ssize_t gpu_busy_percentage_show(struct device *dev, struct device_attribute *attr, char *buf) { struct kgsl_device *device = dev_get_drvdata(dev); return _gpu_busy_show(device, buf); } static ssize_t _min_clock_mhz_show(struct kgsl_device *device, char *buf) { struct kgsl_pwrctrl *pwr = &device->pwrctrl; return scnprintf(buf, PAGE_SIZE, "%d\n", pwr->pwrlevels[pwr->min_pwrlevel].gpu_freq / 1000000); } static ssize_t min_clock_mhz_show(struct device *dev, struct device_attribute *attr, char *buf) { struct kgsl_device *device = dev_get_drvdata(dev); return _min_clock_mhz_show(device, buf); } static ssize_t _min_clock_mhz_store(struct kgsl_device *device, const char *buf, size_t count) { int level, ret; unsigned int freq; struct kgsl_pwrctrl *pwr = &device->pwrctrl; ret = kstrtou32(buf, 0, &freq); if (ret) return ret; freq *= 1000000; level = _get_nearest_pwrlevel(pwr, freq); if (level >= 0) kgsl_pwrctrl_min_pwrlevel_set(device, level); return count; } static ssize_t min_clock_mhz_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct kgsl_device *device = dev_get_drvdata(dev); return _min_clock_mhz_store(device, buf, count); } static ssize_t _max_clock_mhz_show(struct kgsl_device *device, char *buf) { struct kgsl_pwrctrl *pwr = &device->pwrctrl; return scnprintf(buf, PAGE_SIZE, "%d\n", pwr->pwrlevels[pwr->thermal_pwrlevel].gpu_freq / 1000000); } static ssize_t max_clock_mhz_show(struct device *dev, struct device_attribute *attr, char *buf) { struct kgsl_device *device = dev_get_drvdata(dev); return _max_clock_mhz_show(device, buf); } static ssize_t _max_clock_mhz_store(struct kgsl_device *device, const char *buf, size_t count) { u32 freq; int ret, level; ret = kstrtou32(buf, 0, &freq); if (ret) return ret; level = _get_nearest_pwrlevel(&device->pwrctrl, freq * 1000000); if (level < 0) return level; /* * You would think this would set max_pwrlevel but the legacy behavior * is that it set thermal_pwrlevel instead so we don't want to mess with * that. */ ret = kgsl_pwrctrl_set_thermal_limit(device, level); if (ret) return ret; return count; } static ssize_t max_clock_mhz_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct kgsl_device *device = dev_get_drvdata(dev); return _max_clock_mhz_store(device, buf, count); } static ssize_t _clock_mhz_show(struct kgsl_device *device, char *buf) { return scnprintf(buf, PAGE_SIZE, "%ld\n", kgsl_pwrctrl_active_freq(&device->pwrctrl) / 1000000); } static ssize_t clock_mhz_show(struct device *dev, struct device_attribute *attr, char *buf) { struct kgsl_device *device = dev_get_drvdata(dev); return _clock_mhz_show(device, buf); } static ssize_t _freq_table_mhz_show(struct kgsl_device *device, char *buf) { struct kgsl_pwrctrl *pwr = &device->pwrctrl; int index, num_chars = 0; for (index = 0; index < pwr->num_pwrlevels; index++) { num_chars += scnprintf(buf + num_chars, PAGE_SIZE - num_chars - 1, "%d ", pwr->pwrlevels[index].gpu_freq / 1000000); /* One space for trailing null and another for the newline */ if (num_chars >= PAGE_SIZE - 2) break; } buf[num_chars++] = '\n'; return num_chars; } static ssize_t freq_table_mhz_show(struct device *dev, struct device_attribute *attr, char *buf) { struct kgsl_device *device = dev_get_drvdata(dev); return _freq_table_mhz_show(device, buf); } static ssize_t _gpu_tmu_show(struct kgsl_device *device, char *buf) { struct device *dev; struct thermal_zone_device *thermal_dev; int temperature = 0, max_temp = 0; const char *name; struct property *prop; dev = &device->pdev->dev; of_property_for_each_string(dev->of_node, "qcom,tzone-names", prop, name) { thermal_dev = thermal_zone_get_zone_by_name(name); if (IS_ERR(thermal_dev)) continue; if (thermal_zone_get_temp(thermal_dev, &temperature)) continue; max_temp = max(temperature, max_temp); } return scnprintf(buf, PAGE_SIZE, "%d\n", max_temp); } static ssize_t temp_show(struct device *dev, struct device_attribute *attr, char *buf) { struct kgsl_device *device = dev_get_drvdata(dev); return _gpu_tmu_show(device, buf); } static ssize_t pwrscale_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct kgsl_device *device = dev_get_drvdata(dev); int ret; unsigned int enable = 0; ret = kstrtou32(buf, 0, &enable); if (ret) return ret; mutex_lock(&device->mutex); if (enable) kgsl_pwrscale_enable(device); else kgsl_pwrscale_disable(device, false); mutex_unlock(&device->mutex); return count; } static ssize_t pwrscale_show(struct device *dev, struct device_attribute *attr, char *buf) { struct kgsl_device *device = dev_get_drvdata(dev); struct kgsl_pwrscale *psc = &device->pwrscale; return scnprintf(buf, PAGE_SIZE, "%u\n", psc->enabled); } static DEVICE_ATTR_RO(temp); static DEVICE_ATTR_RW(gpuclk); static DEVICE_ATTR_RW(max_gpuclk); static DEVICE_ATTR_RW(idle_timer); static DEVICE_ATTR_RW(minbw_timer); static DEVICE_ATTR_RO(gpubusy); static DEVICE_ATTR_RO(gpu_available_frequencies); static DEVICE_ATTR_RO(gpu_clock_stats); static DEVICE_ATTR_RW(max_pwrlevel); static DEVICE_ATTR_RW(min_pwrlevel); static DEVICE_ATTR_RW(thermal_pwrlevel); static DEVICE_ATTR_RO(num_pwrlevels); static DEVICE_ATTR_RO(reset_count); static DEVICE_ATTR_RW(force_clk_on); static DEVICE_ATTR_RW(force_bus_on); static DEVICE_ATTR_RW(force_rail_on); static DEVICE_ATTR_RW(bus_split); static DEVICE_ATTR_RW(default_pwrlevel); static DEVICE_ATTR_RO(popp); static DEVICE_ATTR_RW(force_no_nap); static DEVICE_ATTR_RO(gpu_busy_percentage); static DEVICE_ATTR_RW(min_clock_mhz); static DEVICE_ATTR_RW(max_clock_mhz); static DEVICE_ATTR_RO(clock_mhz); static DEVICE_ATTR_RO(freq_table_mhz); static DEVICE_ATTR_RW(pwrscale); static const struct attribute *pwrctrl_attr_list[] = { &dev_attr_gpuclk.attr, &dev_attr_max_gpuclk.attr, &dev_attr_idle_timer.attr, &dev_attr_minbw_timer.attr, &dev_attr_gpubusy.attr, &dev_attr_gpu_available_frequencies.attr, &dev_attr_gpu_clock_stats.attr, &dev_attr_max_pwrlevel.attr, &dev_attr_min_pwrlevel.attr, &dev_attr_thermal_pwrlevel.attr, &dev_attr_num_pwrlevels.attr, &dev_attr_reset_count.attr, &dev_attr_force_clk_on.attr, &dev_attr_force_bus_on.attr, &dev_attr_force_rail_on.attr, &dev_attr_force_no_nap.attr, &dev_attr_bus_split.attr, &dev_attr_default_pwrlevel.attr, &dev_attr_popp.attr, &dev_attr_gpu_busy_percentage.attr, &dev_attr_min_clock_mhz.attr, &dev_attr_max_clock_mhz.attr, &dev_attr_clock_mhz.attr, &dev_attr_freq_table_mhz.attr, &dev_attr_temp.attr, &dev_attr_pwrscale.attr, NULL, }; static GPU_SYSFS_ATTR(gpu_busy, 0444, _gpu_busy_show, NULL); static GPU_SYSFS_ATTR(gpu_min_clock, 0644, _min_clock_mhz_show, _min_clock_mhz_store); static GPU_SYSFS_ATTR(gpu_max_clock, 0644, _max_clock_mhz_show, _max_clock_mhz_store); static GPU_SYSFS_ATTR(gpu_clock, 0444, _clock_mhz_show, NULL); static GPU_SYSFS_ATTR(gpu_freq_table, 0444, _freq_table_mhz_show, NULL); static GPU_SYSFS_ATTR(gpu_tmu, 0444, _gpu_tmu_show, NULL); static const struct attribute *gpu_sysfs_attr_list[] = { &gpu_sysfs_attr_gpu_busy.attr, &gpu_sysfs_attr_gpu_min_clock.attr, &gpu_sysfs_attr_gpu_max_clock.attr, &gpu_sysfs_attr_gpu_clock.attr, &gpu_sysfs_attr_gpu_freq_table.attr, &gpu_sysfs_attr_gpu_tmu.attr, NULL, }; int kgsl_pwrctrl_init_sysfs(struct kgsl_device *device) { int ret; ret = sysfs_create_files(&device->dev->kobj, pwrctrl_attr_list); if (ret) return ret; if (!device->gpu_sysfs_kobj.state_in_sysfs) return 0; return sysfs_create_files(&device->gpu_sysfs_kobj, gpu_sysfs_attr_list); } /* * Track the amount of time the gpu is on vs the total system time. * Regularly update the percentage of busy time displayed by sysfs. */ void kgsl_pwrctrl_busy_time(struct kgsl_device *device, u64 time, u64 busy) { struct kgsl_clk_stats *stats = &device->pwrctrl.clk_stats; stats->total += time; stats->busy += busy; if (stats->total < UPDATE_BUSY_VAL) return; /* Update the output regularly and reset the counters. */ stats->total_old = stats->total; stats->busy_old = stats->busy; stats->total = 0; stats->busy = 0; trace_kgsl_gpubusy(device, stats->busy_old, stats->total_old); } static void kgsl_pwrctrl_clk(struct kgsl_device *device, bool state, int requested_state) { struct kgsl_pwrctrl *pwr = &device->pwrctrl; int i = 0; if (gmu_core_gpmu_isenabled(device)) return; if (test_bit(KGSL_PWRFLAGS_CLK_ON, &pwr->ctrl_flags)) return; if (!state) { if (test_and_clear_bit(KGSL_PWRFLAGS_CLK_ON, &pwr->power_flags)) { trace_kgsl_clk(device, state, kgsl_pwrctrl_active_freq(pwr)); /* Disable gpu-bimc-interface clocks */ if (pwr->gpu_bimc_int_clk && pwr->gpu_bimc_interface_enabled) { clk_disable_unprepare(pwr->gpu_bimc_int_clk); pwr->gpu_bimc_interface_enabled = false; } for (i = KGSL_MAX_CLKS - 1; i > 0; i--) clk_disable(pwr->grp_clks[i]); /* High latency clock maintenance. */ if (pwr->pwrlevels[0].gpu_freq > 0) { for (i = KGSL_MAX_CLKS - 1; i > 0; i--) clk_unprepare(pwr->grp_clks[i]); device->ftbl->gpu_clock_set(device, pwr->num_pwrlevels - 1); _isense_clk_set_rate(pwr, pwr->num_pwrlevels - 1); } /* Turn off the IOMMU clocks */ kgsl_mmu_disable_clk(&device->mmu); } else if (requested_state == KGSL_STATE_SLUMBER) { /* High latency clock maintenance. */ for (i = KGSL_MAX_CLKS - 1; i > 0; i--) clk_unprepare(pwr->grp_clks[i]); if ((pwr->pwrlevels[0].gpu_freq > 0)) { device->ftbl->gpu_clock_set(device, pwr->num_pwrlevels - 1); _isense_clk_set_rate(pwr, pwr->num_pwrlevels - 1); } } } else { if (!test_and_set_bit(KGSL_PWRFLAGS_CLK_ON, &pwr->power_flags)) { trace_kgsl_clk(device, state, kgsl_pwrctrl_active_freq(pwr)); if (pwr->pwrlevels[0].gpu_freq > 0) { device->ftbl->gpu_clock_set(device, pwr->active_pwrlevel); _isense_clk_set_rate(pwr, pwr->active_pwrlevel); } for (i = KGSL_MAX_CLKS - 1; i > 0; i--) _gpu_clk_prepare_enable(device, pwr->grp_clks[i], clocks[i]); /* Enable the gpu-bimc-interface clocks */ if (pwr->gpu_bimc_int_clk) { if (pwr->active_pwrlevel == 0 && !pwr->gpu_bimc_interface_enabled) { kgsl_pwrctrl_clk_set_rate( pwr->gpu_bimc_int_clk, pwr->gpu_bimc_int_clk_freq, "bimc_gpu_clk"); _bimc_clk_prepare_enable(device, pwr->gpu_bimc_int_clk, "bimc_gpu_clk"); pwr->gpu_bimc_interface_enabled = true; } } /* Turn on the IOMMU clocks */ kgsl_mmu_enable_clk(&device->mmu); } } } int kgsl_pwrctrl_axi(struct kgsl_device *device, bool state) { struct kgsl_pwrctrl *pwr = &device->pwrctrl; if (test_bit(KGSL_PWRFLAGS_AXI_ON, &pwr->ctrl_flags)) return 0; if (!state) { if (test_and_clear_bit(KGSL_PWRFLAGS_AXI_ON, &pwr->power_flags)) { trace_kgsl_bus(device, state); return kgsl_bus_update(device, KGSL_BUS_VOTE_OFF); } } else { if (!test_and_set_bit(KGSL_PWRFLAGS_AXI_ON, &pwr->power_flags)) { trace_kgsl_bus(device, state); return kgsl_bus_update(device, KGSL_BUS_VOTE_ON); } } return 0; } int kgsl_pwrctrl_enable_cx_gdsc(struct kgsl_device *device) { struct kgsl_pwrctrl *pwr = &device->pwrctrl; struct regulator *regulator = pwr->cx_gdsc; int ret; if (IS_ERR_OR_NULL(regulator)) return 0; ret = wait_for_completion_timeout(&pwr->cx_gdsc_gate, msecs_to_jiffies(5000)); if (!ret) { dev_err(device->dev, "GPU CX wait timeout. Dumping CX votes:\n"); /* Dump the cx regulator consumer list */ qcom_clk_dump(NULL, regulator, false); } ret = regulator_enable(regulator); if (ret) dev_err(device->dev, "Failed to enable CX regulator: %d\n", ret); kgsl_mmu_send_tlb_hint(&device->mmu, false); pwr->cx_gdsc_wait = false; return ret; } static int kgsl_pwtctrl_enable_gx_gdsc(struct kgsl_device *device) { struct regulator *regulator = device->pwrctrl.gx_gdsc; int ret; if (IS_ERR_OR_NULL(regulator)) return 0; ret = regulator_enable(regulator); if (ret) dev_err(device->dev, "Failed to enable GX regulator: %d\n", ret); return ret; } void kgsl_pwrctrl_disable_cx_gdsc(struct kgsl_device *device) { struct regulator *regulator = device->pwrctrl.cx_gdsc; if (IS_ERR_OR_NULL(regulator)) return; kgsl_mmu_send_tlb_hint(&device->mmu, true); reinit_completion(&device->pwrctrl.cx_gdsc_gate); device->pwrctrl.cx_gdsc_wait = true; regulator_disable(regulator); } static void kgsl_pwrctrl_disable_gx_gdsc(struct kgsl_device *device) { struct regulator *regulator = device->pwrctrl.gx_gdsc; if (IS_ERR_OR_NULL(regulator)) return; if (!kgsl_regulator_disable_wait(regulator, 200)) dev_err(device->dev, "Regulator vdd is stuck on\n"); } static int enable_regulators(struct kgsl_device *device) { struct kgsl_pwrctrl *pwr = &device->pwrctrl; int ret; if (test_and_set_bit(KGSL_PWRFLAGS_POWER_ON, &pwr->power_flags)) return 0; ret = kgsl_pwrctrl_enable_cx_gdsc(device); if (!ret) { /* Set parent in retention voltage to power up vdd supply */ ret = kgsl_regulator_set_voltage(device->dev, pwr->gx_gdsc_parent, pwr->gx_gdsc_parent_min_corner); if (!ret) ret = kgsl_pwtctrl_enable_gx_gdsc(device); } if (ret) { clear_bit(KGSL_PWRFLAGS_POWER_ON, &pwr->power_flags); return ret; } trace_kgsl_rail(device, KGSL_PWRFLAGS_POWER_ON); return 0; } int kgsl_pwrctrl_probe_regulators(struct kgsl_device *device, struct platform_device *pdev) { struct kgsl_pwrctrl *pwr = &device->pwrctrl; pwr->cx_gdsc = devm_regulator_get(&pdev->dev, "vddcx"); if (IS_ERR(pwr->cx_gdsc)) { if (PTR_ERR(pwr->cx_gdsc) != -EPROBE_DEFER) dev_err(&pdev->dev, "Couldn't get the vddcx gdsc\n"); return PTR_ERR(pwr->cx_gdsc); } pwr->gx_gdsc = devm_regulator_get(&pdev->dev, "vdd"); if (IS_ERR(pwr->gx_gdsc)) { if (PTR_ERR(pwr->gx_gdsc) != -EPROBE_DEFER) dev_err(&pdev->dev, "Couldn't get the vdd gdsc\n"); return PTR_ERR(pwr->gx_gdsc); } return 0; } static int kgsl_cx_gdsc_event(struct notifier_block *nb, unsigned long event, void *data) { struct kgsl_pwrctrl *pwr = container_of(nb, struct kgsl_pwrctrl, cx_gdsc_nb); struct kgsl_device *device = container_of(pwr, struct kgsl_device, pwrctrl); u32 val; if (!(event & REGULATOR_EVENT_DISABLE) || !pwr->cx_gdsc_wait) return 0; if (pwr->cx_cfg_gdsc_offset) { if (kgsl_regmap_read_poll_timeout(&device->regmap, pwr->cx_cfg_gdsc_offset, val, (val & BIT(15)), 100, 100 * 1000)) dev_err(device->dev, "GPU CX wait timeout.\n"); } pwr->cx_gdsc_wait = false; complete_all(&pwr->cx_gdsc_gate); return 0; } int kgsl_register_gdsc_notifier(struct kgsl_device *device) { struct kgsl_pwrctrl *pwr = &device->pwrctrl; if (!IS_ERR_OR_NULL(pwr->cx_gdsc)) { pwr->cx_gdsc_nb.notifier_call = kgsl_cx_gdsc_event; return devm_regulator_register_notifier(pwr->cx_gdsc, &pwr->cx_gdsc_nb); } return 0; } static int kgsl_pwrctrl_pwrrail(struct kgsl_device *device, bool state) { struct kgsl_pwrctrl *pwr = &device->pwrctrl; int status = 0; if (gmu_core_gpmu_isenabled(device)) return 0; /* * Disabling the regulator means also disabling dependent clocks. * Hence don't disable it if force clock ON is set. */ if (test_bit(KGSL_PWRFLAGS_POWER_ON, &pwr->ctrl_flags) || test_bit(KGSL_PWRFLAGS_CLK_ON, &pwr->ctrl_flags)) return 0; if (!state) { if (test_and_clear_bit(KGSL_PWRFLAGS_POWER_ON, &pwr->power_flags)) { kgsl_mmu_send_tlb_hint(&device->mmu, true); trace_kgsl_rail(device, state); /* Set the parent in retention voltage to disable CPR interrupts */ kgsl_regulator_set_voltage(device->dev, pwr->gx_gdsc_parent, pwr->gx_gdsc_parent_min_corner); kgsl_pwrctrl_disable_gx_gdsc(device); /* Remove the vote for the vdd parent supply */ kgsl_regulator_set_voltage(device->dev, pwr->gx_gdsc_parent, 0); kgsl_pwrctrl_disable_cx_gdsc(device); } } else { status = enable_regulators(device); kgsl_mmu_send_tlb_hint(&device->mmu, false); } return status; } void kgsl_pwrctrl_irq(struct kgsl_device *device, bool state) { struct kgsl_pwrctrl *pwr = &device->pwrctrl; if (!(device->freq_limiter_intr_num || pwr->interrupt_num)) return; if (state) { if (!test_and_set_bit(KGSL_PWRFLAGS_IRQ_ON, &pwr->power_flags)) { trace_kgsl_irq(device, state); if (pwr->interrupt_num > 0) enable_irq(pwr->interrupt_num); if (device->freq_limiter_intr_num > 0) enable_irq(device->freq_limiter_intr_num); } } else { if (test_and_clear_bit(KGSL_PWRFLAGS_IRQ_ON, &pwr->power_flags)) { trace_kgsl_irq(device, state); if (device->freq_limiter_intr_num > 0) disable_irq(device->freq_limiter_intr_num); if (in_interrupt() && (pwr->interrupt_num > 0)) disable_irq_nosync(pwr->interrupt_num); else if (pwr->interrupt_num > 0) disable_irq(pwr->interrupt_num); } } } static int _get_clocks(struct kgsl_device *device) { struct device *dev = &device->pdev->dev; struct kgsl_pwrctrl *pwr = &device->pwrctrl; const char *name; struct property *prop; pwr->isense_clk_indx = 0; of_property_for_each_string(dev->of_node, "clock-names", prop, name) { int i; for (i = 0; i < KGSL_MAX_CLKS; i++) { if (pwr->grp_clks[i] || strcmp(clocks[i], name)) continue; /* apb_pclk should only be enabled if QCOM_KGSL_QDSS_STM is enabled */ if (!strcmp(name, "apb_pclk") && !IS_ENABLED(CONFIG_QCOM_KGSL_QDSS_STM)) continue; pwr->grp_clks[i] = devm_clk_get(dev, name); if (IS_ERR(pwr->grp_clks[i])) { int ret = PTR_ERR(pwr->grp_clks[i]); dev_err(dev, "Couldn't get clock: %s (%d)\n", name, ret); pwr->grp_clks[i] = NULL; return ret; } if (!strcmp(name, "isense_clk")) pwr->isense_clk_indx = i; break; } } if (pwr->isense_clk_indx && of_property_read_u32(dev->of_node, "qcom,isense-clk-on-level", &pwr->isense_clk_on_level)) { dev_err(dev, "Couldn't get isense clock on level\n"); return -ENXIO; } return 0; } static int _isense_clk_set_rate(struct kgsl_pwrctrl *pwr, int level) { int rate; if (!pwr->isense_clk_indx) return -EINVAL; rate = clk_round_rate(pwr->grp_clks[pwr->isense_clk_indx], level > pwr->isense_clk_on_level ? KGSL_XO_CLK_FREQ : KGSL_ISENSE_CLK_FREQ); return kgsl_pwrctrl_clk_set_rate(pwr->grp_clks[pwr->isense_clk_indx], rate, clocks[pwr->isense_clk_indx]); } /* * _gpu_clk_prepare_enable - Enable the specified GPU clock * Try once to enable it and then BUG() for debug */ static void _gpu_clk_prepare_enable(struct kgsl_device *device, struct clk *clk, const char *name) { int ret = clk_prepare_enable(clk); if (ret) dev_err(device->dev, "GPU Clock %s enable error:%d\n", name, ret); } /* * _bimc_clk_prepare_enable - Enable the specified GPU clock * Try once to enable it and then BUG() for debug */ static void _bimc_clk_prepare_enable(struct kgsl_device *device, struct clk *clk, const char *name) { int ret = clk_prepare_enable(clk); /* Failure is fatal so BUG() to facilitate debug */ if (ret) dev_err(device->dev, "GPU clock %s enable error:%d\n", name, ret); } static int kgsl_pwrctrl_clk_set_rate(struct clk *grp_clk, unsigned int freq, const char *name) { int ret = clk_set_rate(grp_clk, freq); WARN(ret, "%s set freq %d failed:%d\n", name, freq, ret); return ret; } int kgsl_pwrctrl_init(struct kgsl_device *device) { int i, result, freq; struct platform_device *pdev = device->pdev; struct kgsl_pwrctrl *pwr = &device->pwrctrl; result = _get_clocks(device); if (result) return result; /* Make sure we have a source clk for freq setting */ if (pwr->grp_clks[0] == NULL) pwr->grp_clks[0] = pwr->grp_clks[1]; /* Getting gfx-bimc-interface-clk frequency */ if (!of_property_read_u32(pdev->dev.of_node, "qcom,gpu-bimc-interface-clk-freq", &pwr->gpu_bimc_int_clk_freq)) pwr->gpu_bimc_int_clk = devm_clk_get(&pdev->dev, "bimc_gpu_clk"); if (pwr->num_pwrlevels == 0) { dev_err(device->dev, "No power levels are defined\n"); return -EINVAL; } init_waitqueue_head(&device->active_cnt_wq); /* Initialize the thermal clock constraints */ pwr->thermal_pwrlevel = 0; pwr->thermal_pwrlevel_floor = pwr->num_pwrlevels - 1; result = dev_pm_qos_add_request(&pdev->dev, &pwr->sysfs_thermal_req, DEV_PM_QOS_MAX_FREQUENCY, PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE); if (result < 0) dev_err(device->dev, "PM QoS thermal request failed:%d\n", result); for (i = 0; i < pwr->num_pwrlevels; i++) { freq = pwr->pwrlevels[i].gpu_freq; if (freq > 0) freq = clk_round_rate(pwr->grp_clks[0], freq); if (freq >= pwr->pwrlevels[i].gpu_freq) pwr->pwrlevels[i].gpu_freq = freq; } clk_set_rate(pwr->grp_clks[0], pwr->pwrlevels[pwr->num_pwrlevels - 1].gpu_freq); freq = clk_round_rate(pwr->grp_clks[6], KGSL_XO_CLK_FREQ); if (freq > 0) kgsl_pwrctrl_clk_set_rate(pwr->grp_clks[6], freq, clocks[6]); _isense_clk_set_rate(pwr, pwr->num_pwrlevels - 1); if (of_property_read_bool(pdev->dev.of_node, "vddcx-supply")) pwr->cx_gdsc = devm_regulator_get(&pdev->dev, "vddcx"); if (of_property_read_bool(pdev->dev.of_node, "vdd-supply")) pwr->gx_gdsc = devm_regulator_get(&pdev->dev, "vdd"); if (of_property_read_bool(pdev->dev.of_node, "vdd-parent-supply")) { pwr->gx_gdsc_parent = devm_regulator_get(&pdev->dev, "vdd-parent"); if (IS_ERR(pwr->gx_gdsc_parent)) { dev_err(device->dev, "Failed to get vdd-parent regulator:%ld\n", PTR_ERR(pwr->gx_gdsc_parent)); return -ENODEV; } if (of_property_read_u32(pdev->dev.of_node, "vdd-parent-min-corner", &pwr->gx_gdsc_parent_min_corner)) { dev_err(device->dev, "vdd-parent-min-corner not found\n"); return -ENODEV; } } init_completion(&pwr->cx_gdsc_gate); complete_all(&pwr->cx_gdsc_gate); result = kgsl_register_gdsc_notifier(device); if (result) { dev_err(&pdev->dev, "Failed to register gdsc notifier: %d\n", result); return result; } pwr->power_flags = 0; pm_runtime_enable(&pdev->dev); return 0; } void kgsl_pwrctrl_close(struct kgsl_device *device) { struct kgsl_pwrctrl *pwr = &device->pwrctrl; pwr->power_flags = 0; if (dev_pm_qos_request_active(&pwr->sysfs_thermal_req)) dev_pm_qos_remove_request(&pwr->sysfs_thermal_req); pm_runtime_disable(&device->pdev->dev); } void kgsl_idle_check(struct work_struct *work) { struct kgsl_device *device = container_of(work, struct kgsl_device, idle_check_ws); int ret = 0; unsigned int requested_state; mutex_lock(&device->mutex); /* * After scheduling idle work for transitioning to SLUMBER, it's * possbile that requested state can change to NONE if any new workload * comes before kgsl_idle_check is executed or it gets the device mutex. * In such case, no need to change state to NONE. */ if (device->requested_state == KGSL_STATE_NONE) { mutex_unlock(&device->mutex); return; } requested_state = device->requested_state; if (device->state == KGSL_STATE_ACTIVE) { if (!atomic_read(&device->active_cnt)) { spin_lock(&device->submit_lock); if (device->submit_now) { spin_unlock(&device->submit_lock); goto done; } /* Don't allow GPU inline submission in SLUMBER */ if (requested_state == KGSL_STATE_SLUMBER) device->skip_inline_submit = true; spin_unlock(&device->submit_lock); ret = kgsl_pwrctrl_change_state(device, device->requested_state); if (ret == -EBUSY) { if (requested_state == KGSL_STATE_SLUMBER) { spin_lock(&device->submit_lock); device->skip_inline_submit = false; spin_unlock(&device->submit_lock); } /* * If the GPU is currently busy, restore * the requested state and reschedule * idle work. */ kgsl_pwrctrl_request_state(device, requested_state); kgsl_schedule_work(&device->idle_check_ws); } } done: if (!ret) kgsl_pwrctrl_request_state(device, KGSL_STATE_NONE); if (device->state == KGSL_STATE_ACTIVE) kgsl_start_idle_timer(device); } kgsl_pwrscale_update(device); mutex_unlock(&device->mutex); } void kgsl_timer(struct timer_list *t) { struct kgsl_device *device = from_timer(device, t, idle_timer); if (device->requested_state != KGSL_STATE_SUSPEND) { kgsl_pwrctrl_request_state(device, KGSL_STATE_SLUMBER); /* Have work run in a non-interrupt context. */ kgsl_schedule_work(&device->idle_check_ws); } } static bool kgsl_pwrctrl_isenabled(struct kgsl_device *device) { struct kgsl_pwrctrl *pwr = &device->pwrctrl; return ((test_bit(KGSL_PWRFLAGS_CLK_ON, &pwr->power_flags) != 0) && (test_bit(KGSL_PWRFLAGS_AXI_ON, &pwr->power_flags) != 0)); } /** * kgsl_pre_hwaccess - Enforce preconditions for touching registers * @device: The device * * This function ensures that the correct lock is held and that the GPU * clock is on immediately before a register is read or written. Note * that this function does not check active_cnt because the registers * must be accessed during device start and stop, when the active_cnt * may legitimately be 0. */ void kgsl_pre_hwaccess(struct kgsl_device *device) { /* In order to touch a register you must hold the device mutex */ WARN_ON(!mutex_is_locked(&device->mutex)); /* * A register access without device power will cause a fatal timeout. * This is not valid for targets with a GMU. */ if (!gmu_core_gpmu_isenabled(device)) WARN_ON(!kgsl_pwrctrl_isenabled(device)); } static int kgsl_pwrctrl_enable(struct kgsl_device *device) { struct kgsl_pwrctrl *pwr = &device->pwrctrl; int level, status; level = pwr->default_pwrlevel; kgsl_pwrctrl_pwrlevel_change(device, level); /* Order pwrrail/clk sequence based upon platform */ status = kgsl_pwrctrl_pwrrail(device, true); if (status) return status; kgsl_pwrctrl_clk(device, true, KGSL_STATE_ACTIVE); kgsl_pwrctrl_axi(device, true); return device->ftbl->regulator_enable(device); } void kgsl_pwrctrl_clear_l3_vote(struct kgsl_device *device) { int status; struct dcvs_freq freq = {0}; if (!device->num_l3_pwrlevels) return; freq.hw_type = DCVS_L3; status = qcom_dcvs_update_votes(KGSL_L3_DEVICE, &freq, 1, DCVS_SLOW_PATH); if (!status) device->cur_l3_pwrlevel = 0; else dev_err(device->dev, "Could not clear l3_vote: %d\n", status); } static void kgsl_pwrctrl_disable(struct kgsl_device *device) { kgsl_pwrctrl_clear_l3_vote(device); /* Order pwrrail/clk sequence based upon platform */ device->ftbl->regulator_disable(device); kgsl_pwrctrl_axi(device, false); kgsl_pwrctrl_clk(device, false, KGSL_STATE_SLUMBER); kgsl_pwrctrl_pwrrail(device, false); } /** * _init() - Get the GPU ready to start, but don't turn anything on * @device - Pointer to the kgsl_device struct */ static int _init(struct kgsl_device *device) { int status = 0; switch (device->state) { case KGSL_STATE_ACTIVE: kgsl_pwrctrl_irq(device, false); del_timer_sync(&device->idle_timer); device->ftbl->stop(device); fallthrough; case KGSL_STATE_AWARE: kgsl_pwrctrl_disable(device); fallthrough; case KGSL_STATE_SLUMBER: fallthrough; case KGSL_STATE_NONE: kgsl_pwrctrl_set_state(device, KGSL_STATE_INIT); } return status; } /** * _wake() - Power up the GPU from a slumber state * @device - Pointer to the kgsl_device struct * * Resume the GPU from a lower power state to ACTIVE. */ static int _wake(struct kgsl_device *device) { struct kgsl_pwrctrl *pwr = &device->pwrctrl; int status = 0; switch (device->state) { case KGSL_STATE_SUSPEND: complete_all(&device->hwaccess_gate); /* Call the GPU specific resume function */ device->ftbl->resume(device); fallthrough; case KGSL_STATE_SLUMBER: status = device->ftbl->start(device, device->pwrctrl.superfast); device->pwrctrl.superfast = false; if (status) { kgsl_pwrctrl_request_state(device, KGSL_STATE_NONE); dev_err(device->dev, "start failed %d\n", status); break; } kgsl_pwrctrl_axi(device, true); kgsl_pwrscale_wake(device); kgsl_pwrctrl_irq(device, true); trace_gpu_frequency( pwr->pwrlevels[pwr->active_pwrlevel].gpu_freq/1000, 0); kgsl_bus_update(device, KGSL_BUS_VOTE_ON); /* Turn on the core clocks */ kgsl_pwrctrl_clk(device, true, KGSL_STATE_ACTIVE); device->ftbl->deassert_gbif_halt(device); pwr->last_stat_updated = ktime_get(); /* * No need to turn on/off irq here as it no longer affects * power collapse */ kgsl_pwrctrl_set_state(device, KGSL_STATE_ACTIVE); /* All settings for power level transitions are complete*/ pwr->previous_pwrlevel = pwr->active_pwrlevel; kgsl_start_idle_timer(device); break; case KGSL_STATE_AWARE: /* Enable state before turning on irq */ kgsl_pwrctrl_set_state(device, KGSL_STATE_ACTIVE); kgsl_pwrctrl_irq(device, true); kgsl_start_idle_timer(device); break; default: dev_warn(device->dev, "unhandled state %s\n", kgsl_pwrstate_to_str(device->state)); kgsl_pwrctrl_request_state(device, KGSL_STATE_NONE); status = -EINVAL; break; } return status; } /* * _aware() - Put device into AWARE * @device: Device pointer * * The GPU should be available for register reads/writes and able * to communicate with the rest of the system. However disable all * paths that allow a switch to an interrupt context (interrupts & * timers). * Return 0 on success else error code */ static int _aware(struct kgsl_device *device) { int status = 0; switch (device->state) { case KGSL_STATE_INIT: status = kgsl_pwrctrl_enable(device); break; case KGSL_STATE_ACTIVE: kgsl_pwrctrl_irq(device, false); del_timer_sync(&device->idle_timer); break; case KGSL_STATE_SLUMBER: status = kgsl_pwrctrl_enable(device); break; default: status = -EINVAL; } if (!status) kgsl_pwrctrl_set_state(device, KGSL_STATE_AWARE); return status; } static int _slumber(struct kgsl_device *device) { int status = 0; switch (device->state) { case KGSL_STATE_ACTIVE: if (!device->ftbl->is_hw_collapsible(device)) { kgsl_pwrctrl_request_state(device, KGSL_STATE_NONE); return -EBUSY; } del_timer_sync(&device->idle_timer); kgsl_pwrctrl_irq(device, false); /* make sure power is on to stop the device*/ status = kgsl_pwrctrl_enable(device); device->ftbl->suspend_context(device); device->ftbl->stop(device); kgsl_pwrctrl_disable(device); kgsl_pwrscale_sleep(device); trace_gpu_frequency(0, 0); kgsl_pwrctrl_set_state(device, KGSL_STATE_SLUMBER); break; case KGSL_STATE_SUSPEND: complete_all(&device->hwaccess_gate); device->ftbl->resume(device); kgsl_pwrctrl_set_state(device, KGSL_STATE_SLUMBER); break; case KGSL_STATE_AWARE: kgsl_pwrctrl_disable(device); trace_gpu_frequency(0, 0); kgsl_pwrctrl_set_state(device, KGSL_STATE_SLUMBER); break; default: kgsl_pwrctrl_request_state(device, KGSL_STATE_NONE); break; } return status; } /* * _suspend() - Put device into suspend * @device: Device pointer * * Return 0 on success else error code */ static int _suspend(struct kgsl_device *device) { int ret = 0; if ((device->state == KGSL_STATE_NONE) || (device->state == KGSL_STATE_INIT) || (device->state == KGSL_STATE_SUSPEND)) return ret; /* * drain to prevent from more commands being submitted * and wait for it to go idle */ ret = device->ftbl->drain_and_idle(device); if (ret) goto err; ret = _slumber(device); if (ret) goto err; kgsl_pwrctrl_set_state(device, KGSL_STATE_SUSPEND); return ret; err: device->ftbl->resume(device); dev_err(device->dev, "device failed to SUSPEND %d\n", ret); return ret; } /* * kgsl_pwrctrl_change_state() changes the GPU state to the input * @device: Pointer to a KGSL device * @state: desired KGSL state * * Caller must hold the device mutex. If the requested state change * is valid, execute it. Otherwise return an error code explaining * why the change has not taken place. Also print an error if an * unexpected state change failure occurs. For example, a change to * SLUMBER may be rejected because the GPU is busy, this is not an error. * A change to SUSPEND should go through no matter what, so if it * fails an additional error message will be printed to dmesg. */ int kgsl_pwrctrl_change_state(struct kgsl_device *device, int state) { int status = 0; if (device->state == state) return status; kgsl_pwrctrl_request_state(device, state); /* Work through the legal state transitions */ switch (state) { case KGSL_STATE_INIT: status = _init(device); break; case KGSL_STATE_AWARE: status = _aware(device); break; case KGSL_STATE_ACTIVE: status = _wake(device); break; case KGSL_STATE_SLUMBER: status = _slumber(device); break; case KGSL_STATE_SUSPEND: status = _suspend(device); break; default: dev_err(device->dev, "bad state request 0x%x\n", state); kgsl_pwrctrl_request_state(device, KGSL_STATE_NONE); status = -EINVAL; break; } return status; } void kgsl_pwrctrl_set_state(struct kgsl_device *device, unsigned int state) { trace_kgsl_pwr_set_state(device, state); device->state = state; device->requested_state = KGSL_STATE_NONE; if (state == KGSL_STATE_SLUMBER) device->pwrctrl.wake_on_touch = false; spin_lock(&device->submit_lock); if (state == KGSL_STATE_ACTIVE) device->skip_inline_submit = false; else device->skip_inline_submit = true; spin_unlock(&device->submit_lock); } void kgsl_pwrctrl_request_state(struct kgsl_device *device, unsigned int state) { if (state != KGSL_STATE_NONE && state != device->requested_state) trace_kgsl_pwr_request_state(device, state); device->requested_state = state; } const char *kgsl_pwrstate_to_str(unsigned int state) { switch (state) { case KGSL_STATE_NONE: return "NONE"; case KGSL_STATE_INIT: return "INIT"; case KGSL_STATE_AWARE: return "AWARE"; case KGSL_STATE_ACTIVE: return "ACTIVE"; case KGSL_STATE_SUSPEND: return "SUSPEND"; case KGSL_STATE_SLUMBER: return "SLUMBER"; default: break; } return "UNKNOWN"; } static int _check_active_count(struct kgsl_device *device, int count) { /* Return 0 if the active count is greater than the desired value */ return atomic_read(&device->active_cnt) > count ? 0 : 1; } int kgsl_active_count_wait(struct kgsl_device *device, int count, unsigned long wait_jiffies) { int result = 0; if (WARN_ON(!mutex_is_locked(&device->mutex))) return -EINVAL; while (atomic_read(&device->active_cnt) > count) { long ret; mutex_unlock(&device->mutex); ret = wait_event_timeout(device->active_cnt_wq, _check_active_count(device, count), wait_jiffies); mutex_lock(&device->mutex); result = ret == 0 ? -ETIMEDOUT : 0; if (!result) wait_jiffies = ret; else break; } return result; } /** * kgsl_pwrctrl_set_default_gpu_pwrlevel() - Set GPU to default power level * @device: Pointer to the kgsl_device struct */ int kgsl_pwrctrl_set_default_gpu_pwrlevel(struct kgsl_device *device) { struct kgsl_pwrctrl *pwr = &device->pwrctrl; unsigned int new_level = pwr->default_pwrlevel; unsigned int old_level = pwr->active_pwrlevel; /* * Update the level according to any thermal, * max/min, or power constraints. */ new_level = kgsl_pwrctrl_adjust_pwrlevel(device, new_level); pwr->active_pwrlevel = new_level; pwr->previous_pwrlevel = old_level; /* Request adjusted DCVS level */ return device->ftbl->gpu_clock_set(device, pwr->active_pwrlevel); } u32 kgsl_pwrctrl_get_acv_perfmode_lvl(struct kgsl_device *device, u32 ddr_freq) { struct kgsl_pwrctrl *pwr = &device->pwrctrl; int i; if (!ddr_freq) return (pwr->ddr_table_count - 1); for (i = 0; i < pwr->ddr_table_count; i++) { if (pwr->ddr_table[i] >= ddr_freq) return i; } /* If DDR frequency is not found, vote perfmode for highest DDR level */ return (pwr->ddr_table_count - 1); } int kgsl_gpu_num_freqs(void) { struct kgsl_device *device = kgsl_get_device(0); if (!device) return -ENODEV; return device->pwrctrl.num_pwrlevels; } EXPORT_SYMBOL(kgsl_gpu_num_freqs); int kgsl_gpu_stat(struct kgsl_gpu_freq_stat *stats, u32 numfreq) { struct kgsl_device *device = kgsl_get_device(0); struct kgsl_pwrctrl *pwr; int i; if (!device) return -ENODEV; pwr = &device->pwrctrl; if (!stats || (numfreq < pwr->num_pwrlevels)) return -EINVAL; mutex_lock(&device->mutex); kgsl_pwrscale_update_stats(device); for (i = 0; i < pwr->num_pwrlevels; i++) { stats[i].freq = pwr->pwrlevels[i].gpu_freq; stats[i].active_time = pwr->clock_times[i]; stats[i].idle_time = pwr->time_in_pwrlevel[i] - pwr->clock_times[i]; } mutex_unlock(&device->mutex); return 0; } EXPORT_SYMBOL(kgsl_gpu_stat);