qjyu
/
OrangePi_CM5
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

RK3588: NPU: Update to 0.9.6 version

This commit is contained in:
yml 2024-07-01 13:47:44 +08:00 committed by baiywt
parent ead859f3d9
commit 57be1fd9cf
21 changed files with 1882 additions and 1088 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
drivers/firmware/rockchip_sip.c
View File

@ -283,6 +283,33 @@ struct arm_smccc_res sip_smc_get_amp_info(u32 sub_func_id, u32 arg1)
}
EXPORT_SYMBOL_GPL(sip_smc_get_amp_info);
struct arm_smccc_res sip_smc_get_pvtpll_info(u32 sub_func_id, u32 arg1)
{
struct arm_smccc_res res;
/*
* res.a0: error code(0: success, !0: error).
* res.a1: low temp config flag(0: support, !0: don't support).
*/
arm_smccc_smc(SIP_PVTPLL_CFG, sub_func_id, arg1, 0, 0, 0, 0, 0, &res);
return res;
}
EXPORT_SYMBOL_GPL(sip_smc_get_pvtpll_info);
struct arm_smccc_res sip_smc_pvtpll_config(u32 sub_func_id, u32 arg1, u32 arg2,
u32 arg3, u32 arg4, u32 arg5, u32 arg6)
{
struct arm_smccc_res res;
/*
* res.a0: error code(0: success, !0: error).
*/
arm_smccc_smc(SIP_PVTPLL_CFG, sub_func_id, arg1, arg2, arg3, arg4, arg5,
arg6, &res);
return res;
}
EXPORT_SYMBOL_GPL(sip_smc_pvtpll_config);
void __iomem *sip_hdcp_request_share_memory(int id)
{
static void __iomem *base;

1
drivers/rknpu/Makefile
View File

@ -10,6 +10,7 @@ rknpu-y += rknpu_reset.o
rknpu-y += rknpu_job.o
rknpu-y += rknpu_debugger.o
rknpu-y += rknpu_iommu.o
rknpu-$(CONFIG_PM_DEVFREQ) += rknpu_devfreq.o
rknpu-$(CONFIG_ROCKCHIP_RKNPU_SRAM) += rknpu_mm.o
rknpu-$(CONFIG_ROCKCHIP_RKNPU_FENCE) += rknpu_fence.o
rknpu-$(CONFIG_ROCKCHIP_RKNPU_DRM_GEM) += rknpu_gem.o

46
drivers/rknpu/include/rknpu_devfreq.h Normal file
View File

@ -0,0 +1,46 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) Rockchip Electronics Co.Ltd
* Author: Finley Xiao <finley.xiao@rock-chips.com>
*/
#ifndef __LINUX_RKNPU_DEVFREQ_H
#define __LINUX_RKNPU_DEVFREQ_H
#ifdef CONFIG_PM_DEVFREQ
void rknpu_devfreq_lock(struct rknpu_device *rknpu_dev);
void rknpu_devfreq_unlock(struct rknpu_device *rknpu_dev);
int rknpu_devfreq_init(struct rknpu_device *rknpu_dev);
void rknpu_devfreq_remove(struct rknpu_device *rknpu_dev);
int rknpu_devfreq_runtime_suspend(struct device *dev);
int rknpu_devfreq_runtime_resume(struct device *dev);
#else
static inline int rknpu_devfreq_init(struct rknpu_device *rknpu_dev)
{
return -EOPNOTSUPP;
}
static inline void rknpu_devfreq_remove(struct rknpu_device *rknpu_dev)
{
}
static inline void rknpu_devfreq_lock(struct rknpu_device *rknpu_dev)
{
}
static inline void rknpu_devfreq_unlock(struct rknpu_device *rknpu_dev)
{
}
static inline int rknpu_devfreq_runtime_suspend(struct device *dev)
{
return -EOPNOTSUPP;
}
static inline int rknpu_devfreq_runtime_resume(struct device *dev)
{
return -EOPNOTSUPP;
}
#endif /* CONFIG_PM_DEVFREQ */
#endif /* __LINUX_RKNPU_DEVFREQ_H_ */

49
drivers/rknpu/include/rknpu_drv.h
View File

@ -10,6 +10,7 @@
#include <linux/completion.h>
#include <linux/device.h>
#include <linux/kref.h>
#include <linux/irq.h>
#include <linux/platform_device.h>
#include <linux/spinlock.h>
#include <linux/regulator/consumer.h>
@ -17,11 +18,9 @@
#include <linux/hrtimer.h>
#include <linux/miscdevice.h>
#ifndef FPGA_PLATFORM
#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
#include <soc/rockchip/rockchip_opp_select.h>
#endif
#endif
#include <soc/rockchip/rockchip_system_monitor.h>
#include <soc/rockchip/rockchip_ipa.h>
#include "rknpu_job.h"
#include "rknpu_fence.h"
@ -30,10 +29,10 @@
#define DRIVER_NAME "rknpu"
#define DRIVER_DESC "RKNPU driver"
#define DRIVER_DATE "20231018"
#define DRIVER_DATE "20240322"
#define DRIVER_MAJOR 0
#define DRIVER_MINOR 9
#define DRIVER_PATCHLEVEL 2
#define DRIVER_PATCHLEVEL 6
#define LOG_TAG "RKNPU"
@ -54,9 +53,18 @@
#define LOG_DEV_DEBUG(dev, fmt, args...) dev_dbg(dev, LOG_TAG ": " fmt, ##args)
#define LOG_DEV_ERROR(dev, fmt, args...) dev_err(dev, LOG_TAG ": " fmt, ##args)
struct rknpu_reset_data {
const char *srst_a_name;
const char *srst_h_name;
#define RKNPU_MAX_IOMMU_DOMAIN_NUM 16
struct rknpu_irqs_data {
const char *name;
irqreturn_t (*irq_hdl)(int irq, void *ctx);
};
struct rknpu_amount_data {
uint16_t offset_clr_all;
uint16_t offset_dt_wr;
uint16_t offset_dt_rd;
uint16_t offset_wt_rd;
};
struct rknpu_config {
@ -68,19 +76,19 @@ struct rknpu_config {
__u32 pc_task_number_mask;
__u32 pc_task_status_offset;
__u32 pc_dma_ctrl;
__u32 bw_enable;
const struct rknpu_irqs_data *irqs;
const struct rknpu_reset_data *resets;
int num_irqs;
int num_resets;
__u64 nbuf_phyaddr;
__u64 nbuf_size;
__u64 max_submit_number;
__u32 core_mask;
const struct rknpu_amount_data *amount_top;
const struct rknpu_amount_data *amount_core;
};
struct rknpu_timer {
__u32 busy_time;
__u32 busy_time_record;
ktime_t busy_time;
ktime_t total_busy_time;
};
struct rknpu_subcore_data {
@ -114,13 +122,14 @@ struct rknpu_device {
spinlock_t irq_lock;
struct mutex power_lock;
struct mutex reset_lock;
struct mutex domain_lock;
struct rknpu_subcore_data subcore_datas[RKNPU_MAX_CORES];
const struct rknpu_config *config;
void __iomem *bw_priority_base;
struct rknpu_fence_context *fence_ctx;
bool iommu_en;
struct reset_control *srst_a[RKNPU_MAX_CORES];
struct reset_control *srst_h[RKNPU_MAX_CORES];
struct reset_control **srsts;
int num_srsts;
struct clk_bulk_data *clks;
int num_clks;
struct regulator *vdd;
@ -130,11 +139,7 @@ struct rknpu_device {
struct thermal_cooling_device *devfreq_cooling;
struct devfreq *devfreq;
unsigned long ondemand_freq;
#ifndef FPGA_PLATFORM
#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
struct rockchip_opp_info opp_info;
#endif
#endif
unsigned long current_freq;
unsigned long current_volt;
int bypass_irq_handler;
@ -161,6 +166,10 @@ struct rknpu_device {
void __iomem *nbuf_base_io;
struct rknpu_mm *sram_mm;
unsigned long power_put_delay;
struct iommu_group *iommu_group;
int iommu_domain_num;
int iommu_domain_id;
struct iommu_domain *iommu_domains[RKNPU_MAX_IOMMU_DOMAIN_NUM];
};
struct rknpu_session {

4
drivers/rknpu/include/rknpu_gem.h
View File

@ -60,6 +60,7 @@ struct rknpu_gem_object {
struct page **pages;
struct sg_table *sgt;
struct drm_mm_node mm_node;
int iommu_domain_id;
};
enum rknpu_cache_type {
@ -71,7 +72,8 @@ enum rknpu_cache_type {
struct rknpu_gem_object *rknpu_gem_object_create(struct drm_device *dev,
unsigned int flags,
unsigned long size,
unsigned long sram_size);
unsigned long sram_size,
int iommu_domain_id);
/* destroy a buffer with gem object */
void rknpu_gem_object_destroy(struct rknpu_gem_object *rknpu_obj);

21
drivers/rknpu/include/rknpu_ioctl.h
View File

@ -31,11 +31,6 @@
#define RKNPU_OFFSET_INT_STATUS 0x28
#define RKNPU_OFFSET_INT_RAW_STATUS 0x2c
#define RKNPU_OFFSET_CLR_ALL_RW_AMOUNT 0x8010
#define RKNPU_OFFSET_DT_WR_AMOUNT 0x8034
#define RKNPU_OFFSET_DT_RD_AMOUNT 0x8038
#define RKNPU_OFFSET_WT_RD_AMOUNT 0x803c
#define RKNPU_OFFSET_ENABLE_MASK 0xf008
#define RKNPU_INT_CLEAR 0x1ffff
@ -134,6 +129,8 @@ enum e_rknpu_action {
RKNPU_POWER_OFF = 21,
RKNPU_GET_TOTAL_SRAM_SIZE = 22,
RKNPU_GET_FREE_SRAM_SIZE = 23,
RKNPU_GET_IOMMU_DOMAIN_ID = 24,
RKNPU_SET_IOMMU_DOMAIN_ID = 25,
};
/**
@ -147,6 +144,8 @@ enum e_rknpu_action {
* @dma_addr: dma address that access by rknpu.
* @sram_size: user-desired sram memory allocation size.
* - this size value would be page-aligned internally.
* @iommu_domain_id: iommu domain id
* @reserved: just padding to be 64-bit aligned.
*/
struct rknpu_mem_create {
__u32 handle;
@ -155,6 +154,8 @@ struct rknpu_mem_create {
__u64 obj_addr;
__u64 dma_addr;
__u64 sram_size;
__s32 iommu_domain_id;
__u32 reserved;
};
/**
@ -249,9 +250,10 @@ struct rknpu_subcore_task {
* @task_counter: task counter
* @priority: submit priority
* @task_obj_addr: address of task object
* @regcfg_obj_addr: address of register config object
* @iommu_domain_id: iommu domain id
* @reserved: just padding to be 64-bit aligned.
* @task_base_addr: task base address
* @user_data: (optional) user data
* @hw_elapse_time: hardware elapse time
* @core_mask: core mask of rknpu
* @fence_fd: dma fence fd
* @subcore_task: subcore task
@ -265,9 +267,10 @@ struct rknpu_submit {
__u32 task_counter;
__s32 priority;
__u64 task_obj_addr;
__u64 regcfg_obj_addr;
__u32 iommu_domain_id;
__u32 reserved;
__u64 task_base_addr;
__u64 user_data;
__s64 hw_elapse_time;
__u32 core_mask;
__s32 fence_fd;
struct rknpu_subcore_task subcore_task[5];

8
drivers/rknpu/include/rknpu_iommu.h
View File

@ -37,4 +37,12 @@ dma_addr_t rknpu_iommu_dma_alloc_iova(struct iommu_domain *domain, size_t size,
void rknpu_iommu_dma_free_iova(struct rknpu_iommu_dma_cookie *cookie,
dma_addr_t iova, size_t size);
int rknpu_iommu_init_domain(struct rknpu_device *rknpu_dev);
int rknpu_iommu_switch_domain(struct rknpu_device *rknpu_dev, int domain_id);
void rknpu_iommu_free_domains(struct rknpu_device *rknpu_dev);
#if KERNEL_VERSION(5, 10, 0) < LINUX_VERSION_CODE
int iommu_get_dma_cookie(struct iommu_domain *domain);
#endif
#endif

4
drivers/rknpu/include/rknpu_job.h
View File

@ -44,9 +44,11 @@ struct rknpu_job {
uint32_t use_core_num;
atomic_t run_count;
atomic_t interrupt_count;
ktime_t hw_commit_time;
ktime_t hw_recoder_time;
ktime_t commit_pc_time;
ktime_t hw_elapse_time;
atomic_t submit_count[RKNPU_MAX_CORES];
int iommu_domain_id;
};
irqreturn_t rknpu_core0_irq_handler(int irq, void *data);

8
drivers/rknpu/include/rknpu_mem.h
View File

@ -37,10 +37,10 @@ struct rknpu_mem_object {
unsigned int owner;
};
int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, unsigned long data,
struct file *file);
int rknpu_mem_destroy_ioctl(struct rknpu_device *rknpu_dev, unsigned long data,
struct file *file);
int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, struct file *file,
unsigned int cmd, unsigned long data);
int rknpu_mem_destroy_ioctl(struct rknpu_device *rknpu_dev, struct file *file,
unsigned long data);
int rknpu_mem_sync_ioctl(struct rknpu_device *rknpu_dev, unsigned long data);
#endif

10
drivers/rknpu/rknpu_debugger.c
View File

@ -46,7 +46,7 @@ static int rknpu_load_show(struct seq_file *m, void *data)
unsigned long flags;
int i;
int load;
uint64_t busy_time_total, div_value;
uint64_t total_busy_time, div_value;
seq_puts(m, "NPU load: ");
for (i = 0; i < rknpu_dev->config->num_irqs; i++) {
@ -57,13 +57,13 @@ static int rknpu_load_show(struct seq_file *m, void *data)
spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
busy_time_total = subcore_data->timer.busy_time_record;
total_busy_time = subcore_data->timer.total_busy_time;
spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
div_value = (RKNPU_LOAD_INTERVAL / 100000);
do_div(busy_time_total, div_value);
load = busy_time_total > 100 ? 100 : busy_time_total;
div_value = (RKNPU_LOAD_INTERVAL / 100);
do_div(total_busy_time, div_value);
load = total_busy_time > 100 ? 100 : total_busy_time;
if (rknpu_dev->config->num_irqs > 1)
seq_printf(m, "%2.d%%,", load);

795
drivers/rknpu/rknpu_devfreq.c Normal file
View File

@ -0,0 +1,795 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) Rockchip Electronics Co.Ltd
* Author: Finley Xiao <finley.xiao@rock-chips.com>
*/
#include <linux/clk.h>
#include <linux/clk-provider.h>
#include <linux/devfreq_cooling.h>
#include <linux/pm_runtime.h>
#include <linux/regmap.h>
#include <../drivers/devfreq/governor.h>
#include "rknpu_drv.h"
#include "rknpu_devfreq.h"
#define POWER_DOWN_FREQ 200000000
static int npu_devfreq_target(struct device *dev, unsigned long *freq,
u32 flags);
static struct monitor_dev_profile npu_mdevp = {
.type = MONITOR_TYPE_DEV,
.low_temp_adjust = rockchip_monitor_dev_low_temp_adjust,
.high_temp_adjust = rockchip_monitor_dev_high_temp_adjust,
#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
.check_rate_volt = rockchip_monitor_check_rate_volt,
#else
.update_volt = rockchip_monitor_check_rate_volt,
#endif
};
static int npu_devfreq_get_dev_status(struct device *dev,
struct devfreq_dev_status *stat)
{
return 0;
}
static int npu_devfreq_get_cur_freq(struct device *dev, unsigned long *freq)
{
struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
*freq = rknpu_dev->current_freq;
return 0;
}
static struct devfreq_dev_profile npu_devfreq_profile = {
.polling_ms = 50,
.target = npu_devfreq_target,
.get_dev_status = npu_devfreq_get_dev_status,
.get_cur_freq = npu_devfreq_get_cur_freq,
};
static int devfreq_rknpu_ondemand_func(struct devfreq *df, unsigned long *freq)
{
struct rknpu_device *rknpu_dev = df->data;
if (rknpu_dev && rknpu_dev->ondemand_freq)
*freq = rknpu_dev->ondemand_freq;
else
*freq = df->previous_freq;
return 0;
}
static int devfreq_rknpu_ondemand_handler(struct devfreq *devfreq,
unsigned int event, void *data)
{
return 0;
}
static struct devfreq_governor devfreq_rknpu_ondemand = {
.name = "rknpu_ondemand",
.get_target_freq = devfreq_rknpu_ondemand_func,
.event_handler = devfreq_rknpu_ondemand_handler,
};
static int rk3576_npu_set_read_margin(struct device *dev,
struct rockchip_opp_info *opp_info,
u32 rm)
{
if (!opp_info->grf || !opp_info->volt_rm_tbl)
return 0;
if (rm == opp_info->current_rm || rm == UINT_MAX)
return 0;
LOG_DEV_DEBUG(dev, "set rm to %d\n", rm);
regmap_write(opp_info->grf, 0x08, 0x001c0000 | (rm << 2));
regmap_write(opp_info->grf, 0x0c, 0x003c0000 | (rm << 2));
regmap_write(opp_info->grf, 0x10, 0x001c0000 | (rm << 2));
return 0;
}
static int rk3588_npu_get_soc_info(struct device *dev, struct device_node *np,
int *bin, int *process)
{
int ret = 0;
u8 value = 0;
if (!bin)
return 0;
if (of_property_match_string(np, "nvmem-cell-names",
"specification_serial_number") >= 0) {
ret = rockchip_nvmem_cell_read_u8(
np, "specification_serial_number", &value);
if (ret) {
LOG_DEV_ERROR(
dev,
"Failed to get specification_serial_number\n");
return ret;
}
/* RK3588M */
if (value == 0xd)
*bin = 1;
/* RK3588J */
else if (value == 0xa)
*bin = 2;
}
if (*bin < 0)
*bin = 0;
LOG_DEV_INFO(dev, "bin=%d\n", *bin);
return ret;
}
#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
static int rk3588_npu_set_soc_info(struct device *dev, struct device_node *np,
struct rockchip_opp_info *opp_info)
{
int bin = opp_info->bin;
if (opp_info->volt_sel < 0)
return 0;
if (bin < 0)
bin = 0;
if (!of_property_read_bool(np, "rockchip,supported-hw"))
return 0;
/* SoC Version */
opp_info->supported_hw[0] = BIT(bin);
/* Speed Grade */
opp_info->supported_hw[1] = BIT(opp_info->volt_sel);
return 0;
}
#else
static int rk3588_npu_set_soc_info(struct device *dev, struct device_node *np,
int bin, int process, int volt_sel)
{
struct opp_table *opp_table;
u32 supported_hw[2];
if (volt_sel < 0)
return 0;
if (bin < 0)
bin = 0;
if (!of_property_read_bool(np, "rockchip,supported-hw"))
return 0;
/* SoC Version */
supported_hw[0] = BIT(bin);
/* Speed Grade */
supported_hw[1] = BIT(volt_sel);
opp_table = dev_pm_opp_set_supported_hw(dev, supported_hw, 2);
if (IS_ERR(opp_table)) {
LOG_DEV_ERROR(dev, "failed to set supported opp\n");
return PTR_ERR(opp_table);
}
return 0;
}
#endif
static int rk3588_npu_set_read_margin(struct device *dev,
struct rockchip_opp_info *opp_info,
u32 rm)
{
u32 offset = 0, val = 0;
int i, ret = 0;
if (!opp_info->grf || !opp_info->volt_rm_tbl)
return 0;
if (rm == opp_info->current_rm || rm == UINT_MAX)
return 0;
LOG_DEV_DEBUG(dev, "set rm to %d\n", rm);
for (i = 0; i < 3; i++) {
ret = regmap_read(opp_info->grf, offset, &val);
if (ret < 0) {
LOG_DEV_ERROR(dev, "failed to get rm from 0x%x\n",
offset);
return ret;
}
val &= ~0x1c;
regmap_write(opp_info->grf, offset, val | (rm << 2));
offset += 4;
}
opp_info->current_rm = rm;
return 0;
}
#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
static int npu_opp_config_regulators(struct device *dev,
struct dev_pm_opp *old_opp,
struct dev_pm_opp *new_opp,
struct regulator **regulators,
unsigned int count)
{
struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
return rockchip_opp_config_regulators(dev, old_opp, new_opp, regulators,
count, &rknpu_dev->opp_info);
}
static int npu_opp_config_clks(struct device *dev, struct opp_table *opp_table,
struct dev_pm_opp *opp, void *data,
bool scaling_down)
{
struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
return rockchip_opp_config_clks(dev, opp_table, opp, data, scaling_down,
&rknpu_dev->opp_info);
}
#endif
static const struct rockchip_opp_data rk3576_npu_opp_data = {
.set_read_margin = rk3576_npu_set_read_margin,
#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
.config_regulators = npu_opp_config_regulators,
.config_clks = npu_opp_config_clks,
#endif
};
static const struct rockchip_opp_data rk3588_npu_opp_data = {
.get_soc_info = rk3588_npu_get_soc_info,
.set_soc_info = rk3588_npu_set_soc_info,
.set_read_margin = rk3588_npu_set_read_margin,
#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
.config_regulators = npu_opp_config_regulators,
.config_clks = npu_opp_config_clks,
#endif
};
static const struct of_device_id rockchip_npu_of_match[] = {
{
.compatible = "rockchip,rk3576",
.data = (void *)&rk3576_npu_opp_data,
},
{
.compatible = "rockchip,rk3588",
.data = (void *)&rk3588_npu_opp_data,
},
{},
};
#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
void rknpu_devfreq_lock(struct rknpu_device *rknpu_dev)
{
if (rknpu_dev->devfreq)
rockchip_opp_dvfs_lock(&rknpu_dev->opp_info);
}
EXPORT_SYMBOL(rknpu_devfreq_lock);
void rknpu_devfreq_unlock(struct rknpu_device *rknpu_dev)
{
if (rknpu_dev->devfreq)
rockchip_opp_dvfs_unlock(&rknpu_dev->opp_info);
}
EXPORT_SYMBOL(rknpu_devfreq_unlock);
static int npu_devfreq_target(struct device *dev, unsigned long *freq,
u32 flags)
{
struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
struct dev_pm_opp *opp;
unsigned long opp_volt;
int ret = 0;
if (!opp_info->is_rate_volt_checked)
return -EINVAL;
opp = devfreq_recommended_opp(dev, freq, flags);
if (IS_ERR(opp))
return PTR_ERR(opp);
opp_volt = dev_pm_opp_get_voltage(opp);
dev_pm_opp_put(opp);
if (*freq == rknpu_dev->current_freq)
return 0;
rockchip_opp_dvfs_lock(opp_info);
if (pm_runtime_active(dev))
opp_info->is_runtime_active = true;
else
opp_info->is_runtime_active = false;
ret = dev_pm_opp_set_rate(dev, *freq);
if (!ret) {
rknpu_dev->current_freq = *freq;
if (rknpu_dev->devfreq)
rknpu_dev->devfreq->last_status.current_frequency =
*freq;
rknpu_dev->current_volt = opp_volt;
LOG_DEV_DEBUG(dev, "set rknpu freq: %lu, volt: %lu\n",
rknpu_dev->current_freq, rknpu_dev->current_volt);
}
rockchip_opp_dvfs_unlock(opp_info);
return ret;
}
static const struct rockchip_opp_data rockchip_npu_opp_data = {
.config_clks = npu_opp_config_clks,
};
int rknpu_devfreq_init(struct rknpu_device *rknpu_dev)
{
struct rockchip_opp_info *info = &rknpu_dev->opp_info;
struct device *dev = rknpu_dev->dev;
struct devfreq_dev_profile *dp;
struct dev_pm_opp *opp;
unsigned int dyn_power_coeff = 0;
int ret = 0;
info->data = &rockchip_npu_opp_data;
rockchip_get_opp_data(rockchip_npu_of_match, info);
ret = rockchip_init_opp_table(dev, info, "clk_npu", "rknpu");
if (ret) {
LOG_DEV_ERROR(dev, "failed to init_opp_table\n");
return -EINVAL;
}
rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
opp = devfreq_recommended_opp(dev, &rknpu_dev->current_freq, 0);
if (IS_ERR(opp)) {
ret = PTR_ERR(opp);
goto err_uinit_table;
}
dev_pm_opp_put(opp);
dp = &npu_devfreq_profile;
dp->initial_freq = rknpu_dev->current_freq;
of_property_read_u32(dev->of_node, "dynamic-power-coefficient",
&dyn_power_coeff);
if (dyn_power_coeff)
dp->is_cooling_device = true;
ret = devfreq_add_governor(&devfreq_rknpu_ondemand);
if (ret) {
LOG_DEV_ERROR(dev, "failed to add rknpu_ondemand governor\n");
goto err_uinit_table;
}
rknpu_dev->devfreq = devm_devfreq_add_device(dev, dp, "rknpu_ondemand",
(void *)rknpu_dev);
if (IS_ERR(rknpu_dev->devfreq)) {
LOG_DEV_ERROR(dev, "failed to add devfreq\n");
ret = PTR_ERR(rknpu_dev->devfreq);
rknpu_dev->devfreq = NULL;
goto err_remove_governor;
}
npu_mdevp.data = rknpu_dev->devfreq;
npu_mdevp.opp_info = &rknpu_dev->opp_info;
rknpu_dev->mdev_info =
rockchip_system_monitor_register(dev, &npu_mdevp);
if (IS_ERR(rknpu_dev->mdev_info)) {
dev_dbg(dev, "without system monitor\n");
rknpu_dev->mdev_info = NULL;
}
rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
rknpu_dev->ondemand_freq = rknpu_dev->current_freq;
rknpu_dev->current_volt = regulator_get_voltage(rknpu_dev->vdd);
rknpu_dev->devfreq->previous_freq = rknpu_dev->current_freq;
if (rknpu_dev->devfreq->suspend_freq)
rknpu_dev->devfreq->resume_freq = rknpu_dev->current_freq;
rknpu_dev->devfreq->last_status.current_frequency =
rknpu_dev->current_freq;
rknpu_dev->devfreq->last_status.total_time = 1;
rknpu_dev->devfreq->last_status.busy_time = 1;
return 0;
err_remove_governor:
devfreq_remove_governor(&devfreq_rknpu_ondemand);
err_uinit_table:
rockchip_uninit_opp_table(dev, info);
return ret;
}
EXPORT_SYMBOL(rknpu_devfreq_init);
int rknpu_devfreq_runtime_suspend(struct device *dev)
{
struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
if (opp_info->is_scmi_clk) {
if (clk_set_rate(opp_info->clk, POWER_DOWN_FREQ))
LOG_DEV_ERROR(dev, "failed to restore clk rate\n");
}
opp_info->current_rm = UINT_MAX;
return 0;
}
EXPORT_SYMBOL(rknpu_devfreq_runtime_suspend);
int rknpu_devfreq_runtime_resume(struct device *dev)
{
struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
int ret = 0;
if (!rknpu_dev->current_freq || !rknpu_dev->current_volt)
return 0;
ret = clk_bulk_prepare_enable(opp_info->nclocks, opp_info->clocks);
if (ret) {
LOG_DEV_INFO(dev, "failed to enable opp clks\n");
return ret;
}
if (opp_info->data && opp_info->data->set_read_margin)
opp_info->data->set_read_margin(dev, opp_info,
opp_info->target_rm);
if (opp_info->is_scmi_clk) {
if (clk_set_rate(opp_info->clk, rknpu_dev->current_freq))
LOG_DEV_ERROR(dev, "failed to set power down rate\n");
}
clk_bulk_disable_unprepare(opp_info->nclocks, opp_info->clocks);
return ret;
}
EXPORT_SYMBOL(rknpu_devfreq_runtime_resume);
#else
void rknpu_devfreq_lock(struct rknpu_device *rknpu_dev)
{
rockchip_monitor_volt_adjust_lock(rknpu_dev->mdev_info);
}
EXPORT_SYMBOL(rknpu_devfreq_lock);
void rknpu_devfreq_unlock(struct rknpu_device *rknpu_dev)
{
rockchip_monitor_volt_adjust_unlock(rknpu_dev->mdev_info);
}
EXPORT_SYMBOL(rknpu_devfreq_unlock);
static int npu_opp_helper(struct dev_pm_set_opp_data *data)
{
struct device *dev = data->dev;
struct dev_pm_opp_supply *old_supply_vdd = &data->old_opp.supplies[0];
struct dev_pm_opp_supply *old_supply_mem = &data->old_opp.supplies[1];
struct dev_pm_opp_supply *new_supply_vdd = &data->new_opp.supplies[0];
struct dev_pm_opp_supply *new_supply_mem = &data->new_opp.supplies[1];
struct regulator *vdd_reg = data->regulators[0];
struct regulator *mem_reg = data->regulators[1];
struct clk *clk = data->clk;
struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
unsigned long old_freq = data->old_opp.rate;
unsigned long new_freq = data->new_opp.rate;
bool is_set_rm = true;
bool is_set_clk = true;
u32 target_rm = UINT_MAX;
int ret = 0;
if (!pm_runtime_active(dev)) {
is_set_rm = false;
if (opp_info->scmi_clk)
is_set_clk = false;
}
ret = clk_bulk_prepare_enable(opp_info->num_clks, opp_info->clks);
if (ret < 0) {
LOG_DEV_ERROR(dev, "failed to enable opp clks\n");
return ret;
}
rockchip_get_read_margin(dev, opp_info, new_supply_vdd->u_volt,
&target_rm);
/* Change frequency */
LOG_DEV_DEBUG(dev, "switching OPP: %lu Hz --> %lu Hz\n", old_freq,
new_freq);
/* Scaling up? Scale voltage before frequency */
if (new_freq >= old_freq) {
rockchip_set_intermediate_rate(dev, opp_info, clk, old_freq,
new_freq, true, is_set_clk);
ret = regulator_set_voltage(mem_reg, new_supply_mem->u_volt,
INT_MAX);
if (ret) {
LOG_DEV_ERROR(dev,
"failed to set volt %lu uV for mem reg\n",
new_supply_mem->u_volt);
goto restore_voltage;
}
ret = regulator_set_voltage(vdd_reg, new_supply_vdd->u_volt,
INT_MAX);
if (ret) {
LOG_DEV_ERROR(dev,
"failed to set volt %lu uV for vdd reg\n",
new_supply_vdd->u_volt);
goto restore_voltage;
}
rockchip_set_read_margin(dev, opp_info, target_rm, is_set_rm);
if (is_set_clk && clk_set_rate(clk, new_freq)) {
ret = -EINVAL;
LOG_DEV_ERROR(dev, "failed to set clk rate: %d\n", ret);
goto restore_rm;
}
/* Scaling down? Scale voltage after frequency */
} else {
rockchip_set_intermediate_rate(dev, opp_info, clk, old_freq,
new_freq, false, is_set_clk);
rockchip_set_read_margin(dev, opp_info, target_rm, is_set_rm);
if (is_set_clk && clk_set_rate(clk, new_freq)) {
ret = -EINVAL;
LOG_DEV_ERROR(dev, "failed to set clk rate: %d\n", ret);
goto restore_rm;
}
ret = regulator_set_voltage(vdd_reg, new_supply_vdd->u_volt,
INT_MAX);
if (ret) {
LOG_DEV_ERROR(dev,
"failed to set volt %lu uV for vdd reg\n",
new_supply_vdd->u_volt);
goto restore_freq;
}
ret = regulator_set_voltage(mem_reg, new_supply_mem->u_volt,
INT_MAX);
if (ret) {
LOG_DEV_ERROR(dev,
"failed to set volt %lu uV for mem reg\n",
new_supply_mem->u_volt);
goto restore_freq;
}
}
clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks);
return 0;
restore_freq:
if (is_set_clk && clk_set_rate(clk, old_freq))
LOG_DEV_ERROR(dev, "failed to restore old-freq %lu Hz\n",
old_freq);
restore_rm:
rockchip_get_read_margin(dev, opp_info, old_supply_vdd->u_volt,
&target_rm);
rockchip_set_read_margin(dev, opp_info, opp_info->current_rm,
is_set_rm);
restore_voltage:
regulator_set_voltage(mem_reg, old_supply_mem->u_volt, INT_MAX);
regulator_set_voltage(vdd_reg, old_supply_vdd->u_volt, INT_MAX);
clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks);
return ret;
}
static int npu_devfreq_target(struct device *dev, unsigned long *freq,
u32 flags)
{
struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
struct dev_pm_opp *opp;
unsigned long opp_volt;
int ret = 0;
if (!npu_mdevp.is_checked)
return -EINVAL;
opp = devfreq_recommended_opp(dev, freq, flags);
if (IS_ERR(opp))
return PTR_ERR(opp);
opp_volt = dev_pm_opp_get_voltage(opp);
dev_pm_opp_put(opp);
rockchip_monitor_volt_adjust_lock(rknpu_dev->mdev_info);
ret = dev_pm_opp_set_rate(dev, *freq);
if (!ret) {
rknpu_dev->current_freq = *freq;
if (rknpu_dev->devfreq)
rknpu_dev->devfreq->last_status.current_frequency =
*freq;
rknpu_dev->current_volt = opp_volt;
LOG_DEV_DEBUG(dev, "set rknpu freq: %lu, volt: %lu\n",
rknpu_dev->current_freq, rknpu_dev->current_volt);
}
rockchip_monitor_volt_adjust_unlock(rknpu_dev->mdev_info);
return ret;
}
static unsigned long npu_get_static_power(struct devfreq *devfreq,
unsigned long voltage)
{
struct device *dev = devfreq->dev.parent;
struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
if (!rknpu_dev->model_data)
return 0;
return rockchip_ipa_get_static_power(rknpu_dev->model_data, voltage);
}
static struct devfreq_cooling_power npu_cooling_power = {
.get_static_power = &npu_get_static_power,
};
int rknpu_devfreq_init(struct rknpu_device *rknpu_dev)
{
struct device *dev = rknpu_dev->dev;
struct devfreq_dev_profile *dp = &npu_devfreq_profile;
struct dev_pm_opp *opp;
struct opp_table *reg_table = NULL;
struct opp_table *opp_table = NULL;
const char *const reg_names[] = { "rknpu", "mem" };
int ret = -EINVAL;
if (strstr(__clk_get_name(rknpu_dev->clks[0].clk), "scmi"))
rknpu_dev->opp_info.scmi_clk = rknpu_dev->clks[0].clk;
if (of_find_property(dev->of_node, "rknpu-supply", NULL) &&
of_find_property(dev->of_node, "mem-supply", NULL)) {
reg_table = dev_pm_opp_set_regulators(dev, reg_names, 2);
if (IS_ERR(reg_table))
return PTR_ERR(reg_table);
opp_table =
dev_pm_opp_register_set_opp_helper(dev, npu_opp_helper);
if (IS_ERR(opp_table)) {
dev_pm_opp_put_regulators(reg_table);
return PTR_ERR(opp_table);
}
} else {
reg_table = dev_pm_opp_set_regulators(dev, reg_names, 1);
if (IS_ERR(reg_table))
return PTR_ERR(reg_table);
}
rockchip_get_opp_data(rockchip_npu_of_match, &rknpu_dev->opp_info);
ret = rockchip_init_opp_table(dev, &rknpu_dev->opp_info, "npu_leakage",
"rknpu");
if (ret) {
LOG_DEV_ERROR(dev, "failed to init_opp_table\n");
return ret;
}
rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
opp = devfreq_recommended_opp(dev, &rknpu_dev->current_freq, 0);
if (IS_ERR(opp)) {
ret = PTR_ERR(opp);
goto err_remove_table;
}
dev_pm_opp_put(opp);
dp->initial_freq = rknpu_dev->current_freq;
ret = devfreq_add_governor(&devfreq_rknpu_ondemand);
if (ret) {
LOG_DEV_ERROR(dev, "failed to add rknpu_ondemand governor\n");
goto err_remove_table;
}
rknpu_dev->devfreq = devm_devfreq_add_device(dev, dp, "rknpu_ondemand",
(void *)rknpu_dev);
if (IS_ERR(rknpu_dev->devfreq)) {
LOG_DEV_ERROR(dev, "failed to add devfreq\n");
ret = PTR_ERR(rknpu_dev->devfreq);
goto err_remove_governor;
}
npu_mdevp.data = rknpu_dev->devfreq;
npu_mdevp.opp_info = &rknpu_dev->opp_info;
rknpu_dev->mdev_info =
rockchip_system_monitor_register(dev, &npu_mdevp);
if (IS_ERR(rknpu_dev->mdev_info)) {
LOG_DEV_DEBUG(dev, "without system monitor\n");
rknpu_dev->mdev_info = NULL;
npu_mdevp.is_checked = true;
}
rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
rknpu_dev->ondemand_freq = rknpu_dev->current_freq;
rknpu_dev->current_volt = regulator_get_voltage(rknpu_dev->vdd);
rknpu_dev->devfreq->previous_freq = rknpu_dev->current_freq;
if (rknpu_dev->devfreq->suspend_freq)
rknpu_dev->devfreq->resume_freq = rknpu_dev->current_freq;
rknpu_dev->devfreq->last_status.current_frequency =
rknpu_dev->current_freq;
rknpu_dev->devfreq->last_status.total_time = 1;
rknpu_dev->devfreq->last_status.busy_time = 1;
of_property_read_u32(dev->of_node, "dynamic-power-coefficient",
(u32 *)&npu_cooling_power.dyn_power_coeff);
rknpu_dev->model_data =
rockchip_ipa_power_model_init(dev, "npu_leakage");
if (IS_ERR_OR_NULL(rknpu_dev->model_data)) {
rknpu_dev->model_data = NULL;
LOG_DEV_ERROR(dev, "failed to initialize power model\n");
} else if (rknpu_dev->model_data->dynamic_coefficient) {
npu_cooling_power.dyn_power_coeff =
rknpu_dev->model_data->dynamic_coefficient;
}
if (!npu_cooling_power.dyn_power_coeff) {
LOG_DEV_ERROR(dev, "failed to get dynamic-coefficient\n");
goto out;
}
rknpu_dev->devfreq_cooling = of_devfreq_cooling_register_power(
dev->of_node, rknpu_dev->devfreq, &npu_cooling_power);
if (IS_ERR_OR_NULL(rknpu_dev->devfreq_cooling))
LOG_DEV_ERROR(dev, "failed to register cooling device\n");
out:
return 0;
err_remove_governor:
devfreq_remove_governor(&devfreq_rknpu_ondemand);
err_remove_table:
rockchip_uninit_opp_table(dev, &rknpu_dev->opp_info);
rknpu_dev->devfreq = NULL;
return ret;
}
EXPORT_SYMBOL(rknpu_devfreq_init);
int rknpu_devfreq_runtime_suspend(struct device *dev)
{
struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
if (opp_info->scmi_clk) {
if (clk_set_rate(opp_info->scmi_clk, POWER_DOWN_FREQ))
LOG_DEV_ERROR(dev, "failed to restore clk rate\n");
}
opp_info->current_rm = UINT_MAX;
return 0;
}
EXPORT_SYMBOL(rknpu_devfreq_runtime_suspend);
int rknpu_devfreq_runtime_resume(struct device *dev)
{
struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
int ret = 0;
if (!rknpu_dev->current_freq || !rknpu_dev->current_volt)
return 0;
ret = clk_bulk_prepare_enable(opp_info->num_clks, opp_info->clks);
if (ret) {
LOG_DEV_INFO(dev, "failed to enable opp clks\n");
return ret;
}
if (opp_info->data && opp_info->data->set_read_margin)
opp_info->data->set_read_margin(dev, opp_info,
opp_info->target_rm);
if (opp_info->scmi_clk) {
if (clk_set_rate(opp_info->scmi_clk, rknpu_dev->current_freq))
LOG_DEV_ERROR(dev, "failed to set power down rate\n");
}
clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks);
return ret;
}
EXPORT_SYMBOL(rknpu_devfreq_runtime_resume);
#endif
void rknpu_devfreq_remove(struct rknpu_device *rknpu_dev)
{
if (rknpu_dev->mdev_info) {
rockchip_system_monitor_unregister(rknpu_dev->mdev_info);
rknpu_dev->mdev_info = NULL;
}
if (rknpu_dev->devfreq)
devfreq_remove_governor(&devfreq_rknpu_ondemand);
rockchip_uninit_opp_table(rknpu_dev->dev, &rknpu_dev->opp_info);
}
EXPORT_SYMBOL(rknpu_devfreq_remove);

986
drivers/rknpu/rknpu_drv.c
View File

File diff suppressed because it is too large Load Diff

65
drivers/rknpu/rknpu_gem.c
View File

@ -15,6 +15,7 @@
#include <linux/iommu.h>
#include <linux/pfn_t.h>
#include <linux/version.h>
#include <linux/version_compat_defs.h>
#include <asm/cacheflush.h>
#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
@ -615,7 +616,8 @@ static void rknpu_gem_free_buf_with_cache(struct rknpu_gem_object *rknpu_obj,
struct rknpu_gem_object *rknpu_gem_object_create(struct drm_device *drm,
unsigned int flags,
unsigned long size,
unsigned long sram_size)
unsigned long sram_size,
int iommu_domain_id)
{
struct rknpu_device *rknpu_dev = drm->dev_private;
struct rknpu_gem_object *rknpu_obj = NULL;
@ -629,6 +631,13 @@ struct rknpu_gem_object *rknpu_gem_object_create(struct drm_device *drm,
remain_ddr_size = round_up(size, PAGE_SIZE);
rknpu_obj = rknpu_gem_init(drm, remain_ddr_size);
if (IS_ERR(rknpu_obj))
return rknpu_obj;
if (!rknpu_iommu_switch_domain(rknpu_dev, iommu_domain_id))
rknpu_obj->iommu_domain_id = iommu_domain_id;
if (!rknpu_dev->iommu_en && (flags & RKNPU_MEM_NON_CONTIGUOUS)) {
/*
* when no IOMMU is available, all allocated buffers are
@ -647,10 +656,6 @@ struct rknpu_gem_object *rknpu_gem_object_create(struct drm_device *drm,
if (sram_size != 0)
sram_size = round_up(sram_size, PAGE_SIZE);
rknpu_obj = rknpu_gem_init(drm, remain_ddr_size);
if (IS_ERR(rknpu_obj))
return rknpu_obj;
/* set memory type and cache attribute from user side. */
rknpu_obj->flags = flags;
@ -687,15 +692,9 @@ struct rknpu_gem_object *rknpu_gem_object_create(struct drm_device *drm,
} else if (IS_ENABLED(CONFIG_NO_GKI) &&
(flags & RKNPU_MEM_TRY_ALLOC_NBUF) &&
rknpu_dev->nbuf_size > 0) {
size_t nbuf_size = 0;
rknpu_obj = rknpu_gem_init(drm, remain_ddr_size);
if (IS_ERR(rknpu_obj))
return rknpu_obj;
nbuf_size = remain_ddr_size <= rknpu_dev->nbuf_size ?
remain_ddr_size :
rknpu_dev->nbuf_size;
size_t nbuf_size = remain_ddr_size <= rknpu_dev->nbuf_size ?
remain_ddr_size :
rknpu_dev->nbuf_size;
/* set memory type and cache attribute from user side. */
rknpu_obj->flags = flags;
@ -712,10 +711,6 @@ struct rknpu_gem_object *rknpu_gem_object_create(struct drm_device *drm,
}
if (remain_ddr_size > 0) {
rknpu_obj = rknpu_gem_init(drm, remain_ddr_size);
if (IS_ERR(rknpu_obj))
return rknpu_obj;
/* set memory type and cache attribute from user side. */
rknpu_obj->flags = flags;
@ -724,13 +719,12 @@ struct rknpu_gem_object *rknpu_gem_object_create(struct drm_device *drm,
goto gem_release;
}
if (rknpu_obj)
LOG_DEBUG(
"created dma addr: %pad, cookie: %p, ddr size: %lu, sram size: %lu, nbuf size: %lu, attrs: %#lx, flags: %#x\n",
&rknpu_obj->dma_addr, rknpu_obj->cookie,
rknpu_obj->size, rknpu_obj->sram_size,
rknpu_obj->nbuf_size, rknpu_obj->dma_attrs,
rknpu_obj->flags);
LOG_DEBUG(
"created dma addr: %pad, cookie: %p, ddr size: %lu, sram size: %lu, nbuf size: %lu, attrs: %#lx, flags: %#x, iommu domain id: %d\n",
&rknpu_obj->dma_addr, rknpu_obj->cookie, rknpu_obj->size,
rknpu_obj->sram_size, rknpu_obj->nbuf_size,
rknpu_obj->dma_attrs, rknpu_obj->flags,
rknpu_obj->iommu_domain_id);
return rknpu_obj;
@ -748,12 +742,15 @@ gem_release:
void rknpu_gem_object_destroy(struct rknpu_gem_object *rknpu_obj)
{
struct drm_gem_object *obj = &rknpu_obj->base;
struct rknpu_device *rknpu_dev = obj->dev->dev_private;
LOG_DEBUG(
"destroy dma addr: %pad, cookie: %p, size: %lu, attrs: %#lx, flags: %#x, handle count: %d\n",
&rknpu_obj->dma_addr, rknpu_obj->cookie, rknpu_obj->size,
rknpu_obj->dma_attrs, rknpu_obj->flags, obj->handle_count);
rknpu_iommu_switch_domain(rknpu_dev, rknpu_obj->iommu_domain_id);
/*
* do not release memory region from exporter.
*
@ -766,8 +763,6 @@ void rknpu_gem_object_destroy(struct rknpu_gem_object *rknpu_obj)
} else {
if (IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) &&
rknpu_obj->sram_size > 0) {
struct rknpu_device *rknpu_dev = obj->dev->dev_private;
if (rknpu_obj->sram_obj != NULL)
rknpu_mm_free(rknpu_dev->sram_mm,
rknpu_obj->sram_obj);
@ -785,7 +780,7 @@ void rknpu_gem_object_destroy(struct rknpu_gem_object *rknpu_obj)
rknpu_gem_release(rknpu_obj);
}
int rknpu_gem_create_ioctl(struct drm_device *dev, void *data,
int rknpu_gem_create_ioctl(struct drm_device *drm, void *data,
struct drm_file *file_priv)
{
struct rknpu_mem_create *args = data;
@ -794,8 +789,9 @@ int rknpu_gem_create_ioctl(struct drm_device *dev, void *data,
rknpu_obj = rknpu_gem_object_find(file_priv, args->handle);
if (!rknpu_obj) {
rknpu_obj = rknpu_gem_object_create(
dev, args->flags, args->size, args->sram_size);
rknpu_obj = rknpu_gem_object_create(drm, args->flags,
args->size, args->sram_size,
args->iommu_domain_id);
if (IS_ERR(rknpu_obj))
return PTR_ERR(rknpu_obj);
@ -831,9 +827,10 @@ int rknpu_gem_map_ioctl(struct drm_device *dev, void *data,
#endif
}
int rknpu_gem_destroy_ioctl(struct drm_device *dev, void *data,
int rknpu_gem_destroy_ioctl(struct drm_device *drm, void *data,
struct drm_file *file_priv)
{
struct rknpu_device *rknpu_dev = drm->dev_private;
struct rknpu_gem_object *rknpu_obj = NULL;
struct rknpu_mem_destroy *args = data;
@ -841,6 +838,8 @@ int rknpu_gem_destroy_ioctl(struct drm_device *dev, void *data,
if (!rknpu_obj)
return -EINVAL;
rknpu_iommu_switch_domain(rknpu_dev, rknpu_obj->iommu_domain_id);
// rknpu_gem_object_put(&rknpu_obj->base);
return rknpu_gem_handle_destroy(file_priv, args->handle);
@ -1044,7 +1043,7 @@ int rknpu_gem_dumb_create(struct drm_file *file_priv, struct drm_device *drm,
else
flags = RKNPU_MEM_CONTIGUOUS | RKNPU_MEM_WRITE_COMBINE;
rknpu_obj = rknpu_gem_object_create(drm, flags, args->size, 0);
rknpu_obj = rknpu_gem_object_create(drm, flags, args->size, 0, 0);
if (IS_ERR(rknpu_obj)) {
LOG_DEV_ERROR(drm->dev, "gem object allocate failed.\n");
return PTR_ERR(rknpu_obj);
@ -1336,7 +1335,7 @@ int rknpu_gem_prime_vmap(struct drm_gem_object *obj, struct iosys_map *map)
return -EINVAL;
vaddr = vmap(rknpu_obj->pages, rknpu_obj->num_pages, VM_MAP,
PAGE_KERNEL);
PAGE_KERNEL);
if (!vaddr)
return -ENOMEM;

193
drivers/rknpu/rknpu_iommu.c
View File

@ -59,3 +59,196 @@ void rknpu_iommu_dma_free_iova(struct rknpu_iommu_dma_cookie *cookie,
free_iova_fast(iovad, iova_pfn(iovad, iova), size >> iova_shift(iovad));
}
#if defined(CONFIG_IOMMU_API) && defined(CONFIG_NO_GKI)
#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
struct iommu_group {
struct kobject kobj;
struct kobject *devices_kobj;
struct list_head devices;
#ifdef __ANDROID_COMMON_KERNEL__
struct xarray pasid_array;
#endif
struct mutex mutex;
void *iommu_data;
void (*iommu_data_release)(void *iommu_data);
char *name;
int id;
struct iommu_domain *default_domain;
struct iommu_domain *blocking_domain;
struct iommu_domain *domain;
struct list_head entry;
unsigned int owner_cnt;
void *owner;
};
#else
struct iommu_group {
struct kobject kobj;
struct kobject *devices_kobj;
struct list_head devices;
struct mutex mutex;
struct blocking_notifier_head notifier;
void *iommu_data;
void (*iommu_data_release)(void *iommu_data);
char *name;
int id;
struct iommu_domain *default_domain;
struct iommu_domain *domain;
struct list_head entry;
};
#endif
int rknpu_iommu_init_domain(struct rknpu_device *rknpu_dev)
{
// init domain 0
if (!rknpu_dev->iommu_domains[0]) {
rknpu_dev->iommu_domain_id = 0;
rknpu_dev->iommu_domains[rknpu_dev->iommu_domain_id] =
iommu_get_domain_for_dev(rknpu_dev->dev);
rknpu_dev->iommu_domain_num = 1;
}
return 0;
}
int rknpu_iommu_switch_domain(struct rknpu_device *rknpu_dev, int domain_id)
{
struct iommu_domain *src_domain = NULL;
struct iommu_domain *dst_domain = NULL;
struct bus_type *bus = NULL;
int src_domain_id = 0;
int ret = -EINVAL;
if (!rknpu_dev->iommu_en)
return -EINVAL;
if (domain_id < 0 || domain_id > (RKNPU_MAX_IOMMU_DOMAIN_NUM - 1)) {
LOG_DEV_ERROR(
rknpu_dev->dev,
"invalid iommu domain id: %d, reuse domain id: %d\n",
domain_id, rknpu_dev->iommu_domain_id);
return -EINVAL;
}
bus = rknpu_dev->dev->bus;
if (!bus)
return -EFAULT;
mutex_lock(&rknpu_dev->domain_lock);
src_domain_id = rknpu_dev->iommu_domain_id;
if (domain_id == src_domain_id) {
mutex_unlock(&rknpu_dev->domain_lock);
return 0;
}
src_domain = iommu_get_domain_for_dev(rknpu_dev->dev);
if (src_domain != rknpu_dev->iommu_domains[src_domain_id]) {
LOG_DEV_ERROR(
rknpu_dev->dev,
"mismatch domain get from iommu_get_domain_for_dev\n");
mutex_unlock(&rknpu_dev->domain_lock);
return -EINVAL;
}
dst_domain = rknpu_dev->iommu_domains[domain_id];
if (dst_domain != NULL) {
iommu_detach_device(src_domain, rknpu_dev->dev);
ret = iommu_attach_device(dst_domain, rknpu_dev->dev);
if (ret) {
LOG_DEV_ERROR(
rknpu_dev->dev,
"failed to attach dst iommu domain, id: %d, ret: %d\n",
domain_id, ret);
if (iommu_attach_device(src_domain, rknpu_dev->dev)) {
LOG_DEV_ERROR(
rknpu_dev->dev,
"failed to reattach src iommu domain, id: %d\n",
src_domain_id);
}
mutex_unlock(&rknpu_dev->domain_lock);
return ret;
}
rknpu_dev->iommu_domain_id = domain_id;
} else {
uint64_t dma_limit = 1ULL << 32;
dst_domain = iommu_domain_alloc(bus);
if (!dst_domain) {
LOG_DEV_ERROR(rknpu_dev->dev,
"failed to allocate iommu domain\n");
mutex_unlock(&rknpu_dev->domain_lock);
return -EIO;
}
// init domain iova_cookie
iommu_get_dma_cookie(dst_domain);
iommu_detach_device(src_domain, rknpu_dev->dev);
ret = iommu_attach_device(dst_domain, rknpu_dev->dev);
if (ret) {
LOG_DEV_ERROR(
rknpu_dev->dev,
"failed to attach iommu domain, id: %d, ret: %d\n",
domain_id, ret);
iommu_domain_free(dst_domain);
mutex_unlock(&rknpu_dev->domain_lock);
return ret;
}
// set domain type to dma domain
dst_domain->type |= __IOMMU_DOMAIN_DMA_API;
// iommu dma init domain
iommu_setup_dma_ops(rknpu_dev->dev, 0, dma_limit);
rknpu_dev->iommu_domain_id = domain_id;
rknpu_dev->iommu_domains[domain_id] = dst_domain;
rknpu_dev->iommu_domain_num++;
}
// reset default iommu domain
rknpu_dev->iommu_group->default_domain = dst_domain;
mutex_unlock(&rknpu_dev->domain_lock);
LOG_INFO("switch iommu domain from %d to %d\n", src_domain_id,
domain_id);
return ret;
}
void rknpu_iommu_free_domains(struct rknpu_device *rknpu_dev)
{
int i = 0;
rknpu_iommu_switch_domain(rknpu_dev, 0);
for (i = 1; i < RKNPU_MAX_IOMMU_DOMAIN_NUM; i++) {
struct iommu_domain *domain = rknpu_dev->iommu_domains[i];
if (domain == NULL)
continue;
iommu_detach_device(domain, rknpu_dev->dev);
iommu_domain_free(domain);
rknpu_dev->iommu_domains[i] = NULL;
}
}
#else
int rknpu_iommu_init_domain(struct rknpu_device *rknpu_dev)
{
return 0;
}
int rknpu_iommu_switch_domain(struct rknpu_device *rknpu_dev, int domain_id)
{
return 0;
}
void rknpu_iommu_free_domains(struct rknpu_device *rknpu_dev)
{
}
#endif

176
drivers/rknpu/rknpu_job.c
View File

@ -14,8 +14,9 @@
#include "rknpu_reset.h"
#include "rknpu_gem.h"
#include "rknpu_fence.h"
#include "rknpu_job.h"
#include "rknpu_mem.h"
#include "rknpu_iommu.h"
#include "rknpu_job.h"
#define _REG_READ(base, offset) readl(base + (offset))
#define _REG_WRITE(base, value, offset) writel(value, base + (offset))
@ -27,7 +28,7 @@ static int rknpu_wait_core_index(int core_mask)
{
int index = 0;
switch (core_mask & ((1 << RKNPU_MAX_CORES) - 1)) {
switch (core_mask) {
case RKNPU_CORE0_MASK:
case RKNPU_CORE0_MASK | RKNPU_CORE1_MASK:
case RKNPU_CORE0_MASK | RKNPU_CORE1_MASK | RKNPU_CORE2_MASK:
@ -73,7 +74,7 @@ static int rknpu_get_task_number(struct rknpu_job *job, int core_index)
int task_num = job->args->task_number;
if (core_index >= RKNPU_MAX_CORES || core_index < 0) {
LOG_ERROR("core_index: %d set error!", core_index);
LOG_ERROR("invalid rknpu core index: %d", core_index);
return 0;
}
@ -128,11 +129,10 @@ static inline struct rknpu_job *rknpu_job_alloc(struct rknpu_device *rknpu_dev,
struct rknpu_submit *args)
{
struct rknpu_job *job = NULL;
int i = 0;
#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
struct rknpu_gem_object *task_obj = NULL;
#endif
if (rknpu_dev->config->num_irqs == 1)
args->core_mask = RKNPU_CORE0_MASK;
job = kzalloc(sizeof(*job), GFP_KERNEL);
if (!job)
@ -145,6 +145,9 @@ static inline struct rknpu_job *rknpu_job_alloc(struct rknpu_device *rknpu_dev,
((args->core_mask & RKNPU_CORE2_MASK) >> 2);
atomic_set(&job->run_count, job->use_core_num);
atomic_set(&job->interrupt_count, job->use_core_num);
job->iommu_domain_id = args->iommu_domain_id;
for (i = 0; i < rknpu_dev->config->num_irqs; i++)
atomic_set(&job->submit_count[i], 0);
#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
task_obj = (struct rknpu_gem_object *)(uintptr_t)args->task_obj_addr;
if (task_obj)
@ -197,19 +200,20 @@ static inline int rknpu_job_wait(struct rknpu_job *job)
break;
if (ret == 0) {
int64_t commit_time = 0;
int64_t elapse_time_us = 0;
spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
commit_time = ktime_us_delta(ktime_get(),
job->commit_pc_time);
elapse_time_us = ktime_us_delta(ktime_get(),
job->hw_commit_time);
continue_wait =
job->commit_pc_time == 0 ?
job->hw_commit_time == 0 ?
true :
(commit_time < args->timeout * 1000);
(elapse_time_us < args->timeout * 1000);
spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
LOG_ERROR(
"job: %p, wait_count: %d, continue_wait: %d, commit time: %lldus, wait time: %lldus, timeout time: %uus\n",
job, wait_count, continue_wait,
(job->commit_pc_time == 0 ? 0 : commit_time),
"job: %p, iommu domain id: %d, wait_count: %d, continue wait: %d, commit elapse time: %lldus, wait time: %lldus, timeout: %uus\n",
job, job->iommu_domain_id, wait_count,
continue_wait,
(job->hw_commit_time == 0 ? 0 : elapse_time_us),
ktime_us_delta(ktime_get(), job->timestamp),
args->timeout * 1000);
}
@ -259,6 +263,7 @@ static inline int rknpu_job_wait(struct rknpu_job *job)
return -EINVAL;
args->task_counter = args->task_number;
args->hw_elapse_time = job->hw_elapse_time;
return 0;
}
@ -368,7 +373,8 @@ static inline int rknpu_job_subcore_commit_pc(struct rknpu_job *job,
return 0;
}
static inline int rknpu_job_subcore_commit(struct rknpu_job *job, int core_index)
static inline int rknpu_job_subcore_commit(struct rknpu_job *job,
int core_index)
{
struct rknpu_device *rknpu_dev = job->rknpu_dev;
struct rknpu_submit *args = job->args;
@ -394,7 +400,7 @@ static inline int rknpu_job_subcore_commit(struct rknpu_job *job, int core_index
static void rknpu_job_commit(struct rknpu_job *job)
{
switch (job->args->core_mask & ((1 << RKNPU_MAX_CORES) - 1)) {
switch (job->args->core_mask) {
case RKNPU_CORE0_MASK:
rknpu_job_subcore_commit(job, 0);
break;
@ -442,8 +448,8 @@ static void rknpu_job_next(struct rknpu_device *rknpu_dev, int core_index)
list_del_init(&job->head[core_index]);
subcore_data->job = job;
job->hw_recoder_time = ktime_get();
job->commit_pc_time = job->hw_recoder_time;
job->hw_commit_time = ktime_get();
job->hw_recoder_time = job->hw_commit_time;
spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
if (atomic_dec_and_test(&job->run_count)) {
@ -455,6 +461,7 @@ static void rknpu_job_done(struct rknpu_job *job, int ret, int core_index)
{
struct rknpu_device *rknpu_dev = job->rknpu_dev;
struct rknpu_subcore_data *subcore_data = NULL;
ktime_t now;
unsigned long flags;
int max_submit_number = rknpu_dev->config->max_submit_number;
@ -470,8 +477,9 @@ static void rknpu_job_done(struct rknpu_job *job, int ret, int core_index)
spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
subcore_data->job = NULL;
subcore_data->task_num -= rknpu_get_task_number(job, core_index);
subcore_data->timer.busy_time +=
ktime_us_delta(ktime_get(), job->hw_recoder_time);
now = ktime_get();
job->hw_elapse_time = ktime_sub(now, job->hw_commit_time);
subcore_data->timer.busy_time += ktime_sub(now, job->hw_recoder_time);
spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
if (atomic_dec_and_test(&job->interrupt_count)) {
@ -495,44 +503,32 @@ static void rknpu_job_done(struct rknpu_job *job, int ret, int core_index)
rknpu_job_next(rknpu_dev, core_index);
}
static int rknpu_schedule_core_index(struct rknpu_device *rknpu_dev)
{
int core_num = rknpu_dev->config->num_irqs;
int task_num = rknpu_dev->subcore_datas[0].task_num;
int core_index = 0;
int i = 0;
for (i = 1; i < core_num; i++) {
if (task_num > rknpu_dev->subcore_datas[i].task_num) {
core_index = i;
task_num = rknpu_dev->subcore_datas[i].task_num;
}
}
return core_index;
}
static void rknpu_job_schedule(struct rknpu_job *job)
{
struct rknpu_device *rknpu_dev = job->rknpu_dev;
struct rknpu_subcore_data *subcore_data = NULL;
int i = 0, core_index = 0;
unsigned long flags;
int task_num_list[3] = { 0, 1, 2 };
int tmp = 0;
if ((job->args->core_mask & ((1 << RKNPU_MAX_CORES) - 1)) ==
RKNPU_CORE_AUTO_MASK) {
if (rknpu_dev->subcore_datas[0].task_num >
rknpu_dev->subcore_datas[1].task_num) {
tmp = task_num_list[1];
task_num_list[1] = task_num_list[0];
task_num_list[0] = tmp;
}
if (rknpu_dev->subcore_datas[task_num_list[0]].task_num >
rknpu_dev->subcore_datas[2].task_num) {
tmp = task_num_list[2];
task_num_list[2] = task_num_list[1];
task_num_list[1] = task_num_list[0];
task_num_list[0] = tmp;
} else if (rknpu_dev->subcore_datas[task_num_list[1]].task_num >
rknpu_dev->subcore_datas[2].task_num) {
tmp = task_num_list[2];
task_num_list[2] = task_num_list[1];
task_num_list[1] = tmp;
}
if (!rknpu_dev->subcore_datas[task_num_list[0]].job)
core_index = task_num_list[0];
else if (!rknpu_dev->subcore_datas[task_num_list[1]].job)
core_index = task_num_list[1];
else if (!rknpu_dev->subcore_datas[task_num_list[2]].job)
core_index = task_num_list[2];
else
core_index = task_num_list[0];
if (job->args->core_mask == RKNPU_CORE_AUTO_MASK) {
core_index = rknpu_schedule_core_index(rknpu_dev);
job->args->core_mask = rknpu_core_mask(core_index);
job->use_core_num = 1;
atomic_set(&job->run_count, job->use_core_num);
@ -749,6 +745,11 @@ static int rknpu_submit(struct rknpu_device *rknpu_dev,
return -EINVAL;
}
if (args->core_mask > rknpu_dev->config->core_mask) {
LOG_ERROR("invalid rknpu core mask: %#x", args->core_mask);
return -EINVAL;
}
job = rknpu_job_alloc(rknpu_dev, args);
if (!job) {
LOG_ERROR("failed to allocate rknpu job!\n");
@ -842,6 +843,8 @@ int rknpu_submit_ioctl(struct drm_device *dev, void *data,
struct rknpu_device *rknpu_dev = dev_get_drvdata(dev->dev);
struct rknpu_submit *args = data;
rknpu_iommu_switch_domain(rknpu_dev, args->iommu_domain_id);
return rknpu_submit(rknpu_dev, args);
}
#endif
@ -890,11 +893,6 @@ int rknpu_get_bw_priority(struct rknpu_device *rknpu_dev, uint32_t *priority,
{
void __iomem *base = rknpu_dev->bw_priority_base;
if (!rknpu_dev->config->bw_enable) {
LOG_WARN("Get bw_priority is not supported on this device!\n");
return 0;
}
if (!base)
return -EINVAL;
@ -919,11 +917,6 @@ int rknpu_set_bw_priority(struct rknpu_device *rknpu_dev, uint32_t priority,
{
void __iomem *base = rknpu_dev->bw_priority_base;
if (!rknpu_dev->config->bw_enable) {
LOG_WARN("Set bw_priority is not supported on this device!\n");
return 0;
}
if (!base)
return -EINVAL;
@ -946,28 +939,41 @@ int rknpu_set_bw_priority(struct rknpu_device *rknpu_dev, uint32_t priority,
int rknpu_clear_rw_amount(struct rknpu_device *rknpu_dev)
{
void __iomem *rknpu_core_base = rknpu_dev->base[0];
const struct rknpu_config *config = rknpu_dev->config;
unsigned long flags;
if (!rknpu_dev->config->bw_enable) {
if (config->amount_top == NULL) {
LOG_WARN("Clear rw_amount is not supported on this device!\n");
return 0;
}
if (rknpu_dev->config->pc_dma_ctrl) {
if (config->pc_dma_ctrl) {
uint32_t pc_data_addr = 0;
spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
pc_data_addr = REG_READ(RKNPU_OFFSET_PC_DATA_ADDR);
REG_WRITE(0x1, RKNPU_OFFSET_PC_DATA_ADDR);
REG_WRITE(0x80000101, RKNPU_OFFSET_CLR_ALL_RW_AMOUNT);
REG_WRITE(0x00000101, RKNPU_OFFSET_CLR_ALL_RW_AMOUNT);
REG_WRITE(0x80000101, config->amount_top->offset_clr_all);
REG_WRITE(0x00000101, config->amount_top->offset_clr_all);
if (config->amount_core) {
REG_WRITE(0x80000101,
config->amount_core->offset_clr_all);
REG_WRITE(0x00000101,
config->amount_core->offset_clr_all);
}
REG_WRITE(pc_data_addr, RKNPU_OFFSET_PC_DATA_ADDR);
spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
} else {
spin_lock(&rknpu_dev->lock);
REG_WRITE(0x80000101, RKNPU_OFFSET_CLR_ALL_RW_AMOUNT);
REG_WRITE(0x00000101, RKNPU_OFFSET_CLR_ALL_RW_AMOUNT);
REG_WRITE(0x80000101, config->amount_top->offset_clr_all);
REG_WRITE(0x00000101, config->amount_top->offset_clr_all);
if (config->amount_core) {
REG_WRITE(0x80000101,
config->amount_core->offset_clr_all);
REG_WRITE(0x00000101,
config->amount_core->offset_clr_all);
}
spin_unlock(&rknpu_dev->lock);
}
@ -978,23 +984,42 @@ int rknpu_get_rw_amount(struct rknpu_device *rknpu_dev, uint32_t *dt_wr,
uint32_t *dt_rd, uint32_t *wd_rd)
{
void __iomem *rknpu_core_base = rknpu_dev->base[0];
int amount_scale = rknpu_dev->config->pc_data_amount_scale;
const struct rknpu_config *config = rknpu_dev->config;
int amount_scale = config->pc_data_amount_scale;
if (!rknpu_dev->config->bw_enable) {
if (config->amount_top == NULL) {
LOG_WARN("Get rw_amount is not supported on this device!\n");
return 0;
}
spin_lock(&rknpu_dev->lock);
if (dt_wr != NULL)
*dt_wr = REG_READ(RKNPU_OFFSET_DT_WR_AMOUNT) * amount_scale;
if (dt_wr != NULL) {
*dt_wr = REG_READ(config->amount_top->offset_dt_wr) *
amount_scale;
if (config->amount_core) {
*dt_wr += REG_READ(config->amount_core->offset_dt_wr) *
amount_scale;
}
}
if (dt_rd != NULL)
*dt_rd = REG_READ(RKNPU_OFFSET_DT_RD_AMOUNT) * amount_scale;
if (dt_rd != NULL) {
*dt_rd = REG_READ(config->amount_top->offset_dt_rd) *
amount_scale;
if (config->amount_core) {
*dt_rd += REG_READ(config->amount_core->offset_dt_rd) *
amount_scale;
}
}
if (wd_rd != NULL)
*wd_rd = REG_READ(RKNPU_OFFSET_WT_RD_AMOUNT) * amount_scale;
if (wd_rd != NULL) {
*wd_rd = REG_READ(config->amount_top->offset_wt_rd) *
amount_scale;
if (config->amount_core) {
*wd_rd += REG_READ(config->amount_core->offset_wt_rd) *
amount_scale;
}
}
spin_unlock(&rknpu_dev->lock);
@ -1003,12 +1028,13 @@ int rknpu_get_rw_amount(struct rknpu_device *rknpu_dev, uint32_t *dt_wr,
int rknpu_get_total_rw_amount(struct rknpu_device *rknpu_dev, uint32_t *amount)
{
const struct rknpu_config *config = rknpu_dev->config;
uint32_t dt_wr = 0;
uint32_t dt_rd = 0;
uint32_t wd_rd = 0;
int ret = -EINVAL;
if (!rknpu_dev->config->bw_enable) {
if (config->amount_top == NULL) {
LOG_WARN(
"Get total_rw_amount is not supported on this device!\n");
return 0;

53
drivers/rknpu/rknpu_mem.c
View File

@ -17,8 +17,8 @@
#ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, unsigned long data,
struct file *file)
int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, struct file *file,
unsigned int cmd, unsigned long data)
{
struct rknpu_mem_create args;
int ret = -EINVAL;
@ -33,14 +33,20 @@ int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, unsigned long data,
struct rknpu_session *session = NULL;
int i, fd;
unsigned int length, page_count;
unsigned int in_size = _IOC_SIZE(cmd);
unsigned int k_size = sizeof(struct rknpu_mem_create);
char *k_data = (char *)&args;
if (unlikely(copy_from_user(&args, (struct rknpu_mem_create *)data,
sizeof(struct rknpu_mem_create)))) {
in_size))) {
LOG_ERROR("%s: copy_from_user failed\n", __func__);
ret = -EFAULT;
return ret;
}
if (k_size > in_size)
memset(k_data + in_size, 0, k_size - in_size);
if (args.flags & RKNPU_MEM_NON_CONTIGUOUS) {
LOG_ERROR("%s: malloc iommu memory unsupported in current!\n",
__func__);
@ -109,22 +115,27 @@ int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, unsigned long data,
__LINE__, &phys, length);
}
page_count = length >> PAGE_SHIFT;
pages = vmalloc(page_count * sizeof(struct page));
if (!pages) {
LOG_ERROR("alloc pages failed\n");
ret = -ENOMEM;
goto err_detach_dma_buf;
}
if (args.flags & RKNPU_MEM_KERNEL_MAPPING) {
page_count = length >> PAGE_SHIFT;
pages = vmalloc(page_count * sizeof(struct page));
if (!pages) {
LOG_ERROR("alloc pages failed\n");
ret = -ENOMEM;
goto err_detach_dma_buf;
}
for (i = 0; i < page_count; i++)
pages[i] = &page[i];
for (i = 0; i < page_count; i++)
pages[i] = &page[i];
rknpu_obj->kv_addr = vmap(pages, page_count, VM_MAP, PAGE_KERNEL);
if (!rknpu_obj->kv_addr) {
LOG_ERROR("vmap pages addr failed\n");
ret = -ENOMEM;
goto err_free_pages;
rknpu_obj->kv_addr =
vmap(pages, page_count, VM_MAP, PAGE_KERNEL);
if (!rknpu_obj->kv_addr) {
LOG_ERROR("vmap pages addr failed\n");
ret = -ENOMEM;
goto err_free_pages;
}
vfree(pages);
pages = NULL;
}
rknpu_obj->size = PAGE_ALIGN(args.size);
@ -142,14 +153,12 @@ int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, unsigned long data,
(__u64)rknpu_obj->dma_addr);
if (unlikely(copy_to_user((struct rknpu_mem_create *)data, &args,
sizeof(struct rknpu_mem_create)))) {
in_size))) {
LOG_ERROR("%s: copy_to_user failed\n", __func__);
ret = -EFAULT;
goto err_unmap_kv_addr;
}
vfree(pages);
pages = NULL;
dma_buf_unmap_attachment(attachment, table, DMA_BIDIRECTIONAL);
dma_buf_detach(dmabuf, attachment);
@ -191,8 +200,8 @@ err_free_obj:
return ret;
}
int rknpu_mem_destroy_ioctl(struct rknpu_device *rknpu_dev, unsigned long data,
struct file *file)
int rknpu_mem_destroy_ioctl(struct rknpu_device *rknpu_dev, struct file *file,
unsigned long data)
{
struct rknpu_mem_object *rknpu_obj, *entry, *q;
struct rknpu_session *session = NULL;

61
drivers/rknpu/rknpu_reset.c
View File

@ -28,27 +28,34 @@ static inline struct reset_control *rknpu_reset_control_get(struct device *dev,
int rknpu_reset_get(struct rknpu_device *rknpu_dev)
{
#ifndef FPGA_PLATFORM
struct reset_control *srst_a = NULL;
struct reset_control *srst_h = NULL;
int i = 0;
int num_srsts = 0;
for (i = 0; i < rknpu_dev->config->num_resets; i++) {
srst_a = rknpu_reset_control_get(
rknpu_dev->dev,
rknpu_dev->config->resets[i].srst_a_name);
if (IS_ERR(srst_a))
return PTR_ERR(srst_a);
rknpu_dev->srst_a[i] = srst_a;
srst_h = rknpu_reset_control_get(
rknpu_dev->dev,
rknpu_dev->config->resets[i].srst_h_name);
if (IS_ERR(srst_h))
return PTR_ERR(srst_h);
rknpu_dev->srst_h[i] = srst_h;
num_srsts = of_count_phandle_with_args(rknpu_dev->dev->of_node,
"resets", "#reset-cells");
if (num_srsts <= 0) {
LOG_DEV_ERROR(rknpu_dev->dev,
"failed to get rknpu resets from dtb\n");
return num_srsts;
}
rknpu_dev->srsts = devm_kcalloc(rknpu_dev->dev, num_srsts,
sizeof(*rknpu_dev->srsts), GFP_KERNEL);
if (!rknpu_dev->srsts)
return -ENOMEM;
for (i = 0; i < num_srsts; ++i) {
rknpu_dev->srsts[i] = devm_reset_control_get_exclusive_by_index(
rknpu_dev->dev, i);
if (IS_ERR(rknpu_dev->srsts[i])) {
rknpu_dev->num_srsts = i;
return PTR_ERR(rknpu_dev->srsts[i]);
}
}
rknpu_dev->num_srsts = num_srsts;
return num_srsts;
#endif
return 0;
@ -93,7 +100,7 @@ int rknpu_soft_reset(struct rknpu_device *rknpu_dev)
#ifndef FPGA_PLATFORM
struct iommu_domain *domain = NULL;
struct rknpu_subcore_data *subcore_data = NULL;
int ret = -EINVAL, i = 0;
int ret = 0, i = 0;
if (rknpu_dev->bypass_soft_reset) {
LOG_WARN("bypass soft reset\n");
@ -112,17 +119,17 @@ int rknpu_soft_reset(struct rknpu_device *rknpu_dev)
wake_up(&subcore_data->job_done_wq);
}
LOG_INFO("soft reset\n");
LOG_INFO("soft reset, num: %d\n", rknpu_dev->num_srsts);
for (i = 0; i < rknpu_dev->config->num_resets; i++) {
ret = rknpu_reset_assert(rknpu_dev->srst_a[i]);
ret |= rknpu_reset_assert(rknpu_dev->srst_h[i]);
for (i = 0; i < rknpu_dev->num_srsts; ++i)
ret |= rknpu_reset_assert(rknpu_dev->srsts[i]);
udelay(10);
udelay(10);
ret |= rknpu_reset_deassert(rknpu_dev->srst_a[i]);
ret |= rknpu_reset_deassert(rknpu_dev->srst_h[i]);
}
for (i = 0; i < rknpu_dev->num_srsts; ++i)
ret |= rknpu_reset_deassert(rknpu_dev->srsts[i]);
udelay(10);
if (ret) {
LOG_DEV_ERROR(rknpu_dev->dev,

228
drivers/soc/rockchip/rockchip_opp_select.c
View File

@ -12,6 +12,7 @@
#include <linux/nvmem-consumer.h>
#include <linux/regmap.h>
#include <linux/regulator/consumer.h>
#include <linux/rockchip/rockchip_sip.h>
#include <linux/slab.h>
#include <linux/soc/rockchip/pvtm.h>
#include <linux/thermal.h>
@ -1004,15 +1005,15 @@ void rockchip_pvtpll_add_length(struct rockchip_opp_info *info)
}
if (of_property_read_u32(np, "rockchip,pvtpll-len-min-rate", &min_rate))
return;
goto out;
if (of_property_read_u32(np, "rockchip,pvtpll-len-max-rate", &max_rate))
return;
goto out;
if (of_property_read_u32(np, "rockchip,pvtpll-len-margin", &margin))
return;
goto out;
opp_table = dev_pm_opp_get_opp_table(info->dev);
if (!opp_table)
return;
goto out;
old_rate = clk_get_rate(opp_table->clk);
opp_flag = OPP_ADD_LENGTH | ((margin & OPP_LENGTH_MASK) << OPP_LENGTH_SHIFT);
@ -1033,9 +1034,98 @@ void rockchip_pvtpll_add_length(struct rockchip_opp_info *info)
clk_set_rate(opp_table->clk, old_rate);
dev_pm_opp_put_opp_table(opp_table);
out:
of_node_put(np);
}
EXPORT_SYMBOL(rockchip_pvtpll_add_length);
void rockchip_init_pvtpll_table(struct rockchip_opp_info *info, int bin)
{
struct device_node *np = NULL;
struct property *prop = NULL;
struct of_phandle_args clkspec = { 0 };
struct arm_smccc_res res;
char prop_name[NAME_MAX];
u32 *value;
int count;
int ret, i;
if (!info)
return;
np = of_parse_phandle(info->dev->of_node, "operating-points-v2", 0);
if (!np) {
dev_warn(info->dev, "OPP-v2 not supported\n");
return;
}
ret = of_parse_phandle_with_args(info->dev->of_node, "clocks",
"#clock-cells", 0, &clkspec);
if (ret)
goto out;
info->pvtpll_clk_id = clkspec.args[0];
of_node_put(clkspec.np);
res = sip_smc_get_pvtpll_info(PVTPLL_GET_INFO, info->pvtpll_clk_id);
if (res.a0)
goto out;
if (!res.a1)
info->pvtpll_low_temp = true;
if (bin > 0) {
snprintf(prop_name, sizeof(prop_name),
"rockchip,pvtpll-table-B%d", bin);
prop = of_find_property(np, prop_name, NULL);
}
if (!prop)
sprintf(prop_name, "rockchip,pvtpll-table");
prop = of_find_property(np, prop_name, NULL);
if (!prop)
goto out;
count = of_property_count_u32_elems(np, prop_name);
if (count < 0) {
dev_err(info->dev, "%s: Invalid %s property (%d)\n",
__func__, prop_name, count);
goto out;
} else if (count % 5) {
dev_err(info->dev, "Invalid count of %s\n", prop_name);
goto out;
}
value = kmalloc_array(count, sizeof(*value), GFP_KERNEL);
if (!value)
goto out;
ret = of_property_read_u32_array(np, prop_name, value, count);
if (ret) {
dev_err(info->dev, "%s: error parsing %s: %d\n",
__func__, prop_name, ret);
goto free_value;
}
for (i = 0; i < count; i += 5) {
res = sip_smc_pvtpll_config(PVTPLL_ADJUST_TABLE,
info->pvtpll_clk_id, value[i],
value[i + 1], value[i + 2],
value[i + 3], value[i + 4]);
if (res.a0) {
dev_err(info->dev,
"%s: error cfg clk_id=%u %u %u %u %u %u (%d)\n",
__func__, info->pvtpll_clk_id, value[i],
value[i + 1], value[i + 2], value[i + 3],
value[i + 4], (int)res.a0);
goto free_value;
}
}
free_value:
kfree(value);
out:
of_node_put(np);
}
EXPORT_SYMBOL(rockchip_init_pvtpll_table);
static int rockchip_get_pvtm_pvtpll(struct device *dev, struct device_node *np,
char *reg_name)
{
@ -1186,12 +1276,17 @@ void rockchip_of_get_pvtm_sel(struct device *dev, struct device_node *np,
snprintf(name, sizeof(name),
"rockchip,p%d-pvtm-voltage-sel", process);
prop = of_find_property(np, name, NULL);
} else if (bin >= 0) {
} else if (bin > 0) {
of_property_read_u32(np, "rockchip,pvtm-hw", &hw);
if (hw && (hw & BIT(bin))) {
sprintf(name, "rockchip,pvtm-voltage-sel-hw");
prop = of_find_property(np, name, NULL);
}
if (!prop) {
snprintf(name, sizeof(name),
"rockchip,pvtm-voltage-sel-B%d", bin);
prop = of_find_property(np, name, NULL);
}
}
if (!prop)
sprintf(name, "rockchip,pvtm-voltage-sel");
@ -1633,6 +1728,10 @@ int rockchip_adjust_power_scale(struct device *dev, int scale)
rockchip_adjust_opp_by_irdrop(dev, np, &safe_rate, &max_rate);
dev_info(dev, "avs=%d\n", avs);
if (!safe_rate && !scale)
goto out_np;
clk = of_clk_get_by_name(np, NULL);
if (IS_ERR(clk)) {
if (!safe_rate)
@ -1646,14 +1745,14 @@ int rockchip_adjust_power_scale(struct device *dev, int scale)
if (safe_rate)
irdrop_scale = rockchip_pll_clk_rate_to_scale(clk, safe_rate);
if (max_rate)
opp_scale = rockchip_pll_clk_rate_to_scale(clk, max_rate);
target_scale = max(irdrop_scale, scale);
if (target_scale <= 0)
goto out_clk;
dev_dbg(dev, "target_scale=%d, irdrop_scale=%d, scale=%d\n",
target_scale, irdrop_scale, scale);
if (max_rate)
opp_scale = rockchip_pll_clk_rate_to_scale(clk, max_rate);
if (avs == AVS_SCALING_RATE) {
ret = rockchip_pll_clk_adaptive_scaling(clk, target_scale);
if (ret)
@ -1838,13 +1937,78 @@ int rockchip_set_intermediate_rate(struct device *dev,
}
EXPORT_SYMBOL(rockchip_set_intermediate_rate);
static int rockchip_get_opp_clk(struct device *dev, struct device_node *np,
struct rockchip_opp_info *info)
{
struct clk_bulk_data *clks;
struct of_phandle_args clkspec;
int ret = 0, num_clks = 0, i;
if (of_find_property(np, "rockchip,opp-clocks", NULL)) {
num_clks = of_count_phandle_with_args(np, "rockchip,opp-clocks",
"#clock-cells");
if (num_clks <= 0)
return 0;
clks = devm_kcalloc(dev, num_clks, sizeof(*clks), GFP_KERNEL);
if (!clks)
return -ENOMEM;
for (i = 0; i < num_clks; i++) {
ret = of_parse_phandle_with_args(np,
"rockchip,opp-clocks",
"#clock-cells", i,
&clkspec);
if (ret < 0) {
dev_err(dev, "%s: failed to parse opp clk %d\n",
np->name, i);
goto error;
}
clks[i].clk = of_clk_get_from_provider(&clkspec);
of_node_put(clkspec.np);
if (IS_ERR(clks[i].clk)) {
ret = PTR_ERR(clks[i].clk);
clks[i].clk = NULL;
dev_err(dev, "%s: failed to get opp clk %d\n",
np->name, i);
goto error;
}
}
} else {
num_clks = of_clk_get_parent_count(np);
if (num_clks <= 0)
return 0;
clks = devm_kcalloc(dev, num_clks, sizeof(*clks), GFP_KERNEL);
if (!clks)
return -ENOMEM;
for (i = 0; i < num_clks; i++) {
clks[i].clk = of_clk_get(np, i);
if (IS_ERR(clks[i].clk)) {
ret = PTR_ERR(clks[i].clk);
clks[i].clk = NULL;
dev_err(dev, "%s: failed to get clk %d\n",
np->name, i);
goto error;
}
}
}
info->clks = clks;
info->num_clks = num_clks;
return 0;
error:
while (--i >= 0)
clk_put(clks[i].clk);
devm_kfree(dev, clks);
return ret;
}
int rockchip_init_opp_table(struct device *dev, struct rockchip_opp_info *info,
char *lkg_name, char *reg_name)
{
struct device_node *np;
int bin = -EINVAL, process = -EINVAL;
int scale = 0, volt_sel = -EINVAL;
int ret = 0, num_clks = 0, i;
int ret = 0;
u32 freq;
/* Get OPP descriptor node */
@ -1857,24 +2021,10 @@ int rockchip_init_opp_table(struct device *dev, struct rockchip_opp_info *info,
goto next;
info->dev = dev;
num_clks = of_clk_get_parent_count(np);
if (num_clks > 0) {
info->clks = devm_kcalloc(dev, num_clks, sizeof(*info->clks),
GFP_KERNEL);
if (!info->clks) {
ret = -ENOMEM;
goto out;
}
for (i = 0; i < num_clks; i++) {
info->clks[i].clk = of_clk_get(np, i);
if (IS_ERR(info->clks[i].clk)) {
ret = PTR_ERR(info->clks[i].clk);
dev_err(dev, "%s: failed to get clk %d\n",
np->name, i);
goto out;
}
}
info->num_clks = num_clks;
ret = rockchip_get_opp_clk(dev, np, info);
if (ret)
goto out;
if (info->clks) {
ret = clk_bulk_prepare_enable(info->num_clks, info->clks);
if (ret) {
dev_err(dev, "failed to enable opp clks\n");
@ -1900,6 +2050,7 @@ int rockchip_init_opp_table(struct device *dev, struct rockchip_opp_info *info,
next:
rockchip_get_soc_info(dev, np, &bin, &process);
rockchip_init_pvtpll_table(info, bin);
rockchip_get_scale_volt_sel(dev, lkg_name, reg_name, bin, process,
&scale, &volt_sel);
if (info && info->data && info->data->set_soc_info)
@ -1925,6 +2076,31 @@ out:
}
EXPORT_SYMBOL(rockchip_init_opp_table);
void rockchip_uninit_opp_table(struct device *dev, struct rockchip_opp_info *info)
{
struct opp_table *opp_table;
if (info) {
kfree(info->opp_table);
info->opp_table = NULL;
devm_kfree(dev, info->clks);
info->clks = NULL;
devm_kfree(dev, info->volt_rm_tbl);
info->volt_rm_tbl = NULL;
}
opp_table = dev_pm_opp_get_opp_table(dev);
if (IS_ERR(opp_table))
return;
dev_pm_opp_of_remove_table(dev);
if (opp_table->prop_name)
dev_pm_opp_put_prop_name(opp_table);
if (opp_table->supported_hw)
dev_pm_opp_put_supported_hw(opp_table);
dev_pm_opp_put_opp_table(opp_table);
}
EXPORT_SYMBOL(rockchip_uninit_opp_table);
MODULE_DESCRIPTION("ROCKCHIP OPP Select");
MODULE_AUTHOR("Finley Xiao <finley.xiao@rock-chips.com>, Liang Chen <cl@rock-chips.com>");
MODULE_LICENSE("GPL");

191
drivers/video/rockchip/mpp/mpp_rkvenc2.c
View File

@ -32,6 +32,7 @@
#include <soc/rockchip/rockchip_ipa.h>
#include <soc/rockchip/rockchip_opp_select.h>
#include <soc/rockchip/rockchip_system_monitor.h>
#include <soc/rockchip/rockchip_iommu.h>
#include "mpp_debug.h"
#include "mpp_iommu.h"
@ -44,6 +45,8 @@
#define RKVENC_MAX_DCHS_ID 4
#define RKVENC_MAX_SLICE_FIFO_LEN 256
#define RKVENC_SCLR_DONE_STA BIT(2)
#define RKVENC_WDG 0x38
#define TIMEOUT_MS 100
#define to_rkvenc_info(info) \
container_of(info, struct rkvenc_hw_info, hw)
@ -128,6 +131,11 @@ struct rkvenc_hw_info {
#define INT_STA_RBUS_ERR_STA BIT(7)
#define INT_STA_WDG_STA BIT(8)
#define INT_STA_ERROR (INT_STA_BRSP_OTSD_STA | \
INT_STA_WBUS_ERR_STA | \
INT_STA_RBUS_ERR_STA | \
INT_STA_WDG_STA)
#define DCHS_REG_OFFSET (0x304)
#define DCHS_CLASS_OFFSET (33)
#define DCHS_TXE (0x10)
@ -294,6 +302,7 @@ struct rkvenc_dev {
#ifdef CONFIG_PM_DEVFREQ
struct rockchip_opp_info opp_info;
struct monitor_dev_info *mdev_info;
struct opp_table *opp_table;
#endif
};
@ -1194,6 +1203,7 @@ static int rkvenc_run(struct mpp_dev *mpp, struct mpp_task *mpp_task)
struct rkvenc_task *task = to_rkvenc_task(mpp_task);
struct rkvenc_hw_info *hw = enc->hw_info;
u32 timing_en = mpp->srv->timing_en;
u32 timeout_thd;
mpp_debug_enter();
@ -1242,11 +1252,18 @@ static int rkvenc_run(struct mpp_dev *mpp, struct mpp_task *mpp_task)
/* init current task */
mpp->cur_task = mpp_task;
/*
* reconfig timeout threshold.
* bit0-bit23,x1024 core clk cycles
*/
timeout_thd = mpp_read(mpp, RKVENC_WDG) & 0xff000000;
timeout_thd |= TIMEOUT_MS * clk_get_rate(enc->core_clk_info.clk) / 1024000;
mpp_write(mpp, RKVENC_WDG, timeout_thd);
mpp_task_run_begin(mpp_task, timing_en, MPP_WORK_TIMEOUT_DELAY);
/* Flush the register before the start the device */
wmb();
mpp_write(mpp, enc->hw_info->enc_start_base, start_val);
mpp_task_run_end(mpp_task, timing_en);
@ -1256,9 +1273,9 @@ static int rkvenc_run(struct mpp_dev *mpp, struct mpp_task *mpp_task)
return 0;
}
static void rkvenc2_read_slice_len(struct mpp_dev *mpp, struct rkvenc_task *task)
static void rkvenc2_read_slice_len(struct mpp_dev *mpp, struct rkvenc_task *task,
u32 last)
{
u32 last = mpp_read_relaxed(mpp, 0x002c) & INT_STA_ENC_DONE_STA;
u32 sli_num = mpp_read_relaxed(mpp, RKVENC2_REG_SLICE_NUM_BASE);
union rkvenc2_slice_len_info slice_info;
u32 task_id = task->mpp_task.task_id;
@ -1298,46 +1315,49 @@ static int rkvenc_irq(struct mpp_dev *mpp)
struct rkvenc_hw_info *hw = enc->hw_info;
struct mpp_task *mpp_task = NULL;
struct rkvenc_task *task = NULL;
u32 int_clear = 1;
u32 irq_mask = 0;
u32 irq_status;
int ret = IRQ_NONE;
mpp_debug_enter();
mpp->irq_status = mpp_read(mpp, hw->int_sta_base);
if (!mpp->irq_status)
irq_status = mpp_read(mpp, hw->int_sta_base);
mpp_debug(DEBUG_IRQ_STATUS, "%s irq_status: %08x\n",
dev_name(mpp->dev), irq_status);
if (!irq_status)
return ret;
/* clear int first */
mpp_write(mpp, hw->int_clr_base, irq_status);
/*
* prevent watch dog irq storm.
* The encoder did not stop working when watchdog interrupt is triggered,
* it still check timeout and trigger watch dog irq.
*/
if (irq_status & INT_STA_WDG_STA)
mpp_write(mpp, hw->int_mask_base, INT_STA_WDG_STA);
if (mpp->cur_task) {
mpp_task = mpp->cur_task;
task = to_rkvenc_task(mpp_task);
}
if (mpp->irq_status & INT_STA_ENC_DONE_STA) {
if (task) {
if (task->task_split)
rkvenc2_read_slice_len(mpp, task);
/* 1. read slice number and slice length */
if (task && task->task_split &&
(irq_status & (INT_STA_SLC_DONE_STA | INT_STA_ENC_DONE_STA))) {
mpp_time_part_diff(mpp_task);
rkvenc2_read_slice_len(mpp, task, irq_status & INT_STA_ENC_DONE_STA);
wake_up(&mpp_task->wait);
}
wake_up(&mpp_task->wait);
}
/* 2. process slice irq */
if (irq_status & INT_STA_SLC_DONE_STA)
ret = IRQ_HANDLED;
irq_mask = INT_STA_ENC_DONE_STA;
ret = IRQ_WAKE_THREAD;
if (enc->bs_overflow) {
mpp->irq_status |= INT_STA_BSF_OFLW_STA;
enc->bs_overflow = 0;
}
} else if (mpp->irq_status & INT_STA_SLC_DONE_STA) {
if (task && task->task_split) {
mpp_time_part_diff(mpp_task);
rkvenc2_read_slice_len(mpp, task);
wake_up(&mpp_task->wait);
}
irq_mask = INT_STA_ENC_DONE_STA;
int_clear = 0;
} else if (mpp->irq_status & INT_STA_BSF_OFLW_STA) {
/* 3. process bitstream overflow */
if (irq_status & INT_STA_BSF_OFLW_STA) {
u32 bs_rd = mpp_read(mpp, RKVENC2_REG_ADR_BSBR);
u32 bs_wr = mpp_read(mpp, RKVENC2_REG_ST_BSB);
u32 bs_top = mpp_read(mpp, RKVENC2_REG_ADR_BSBT);
@ -1349,28 +1369,33 @@ static int rkvenc_irq(struct mpp_dev *mpp)
bs_wr += 128;
if (bs_wr >= bs_top)
bs_wr = bs_bot;
/* clear int first */
mpp_write(mpp, hw->int_clr_base, mpp->irq_status);
/* update write addr for enc continue */
mpp_write(mpp, RKVENC2_REG_ADR_BSBS, bs_wr);
enc->bs_overflow = 1;
irq_mask = 0;
int_clear = 0;
ret = IRQ_HANDLED;
} else {
dev_err(mpp->dev, "found error status %08x\n", mpp->irq_status);
irq_mask = mpp->irq_status;
ret = IRQ_HANDLED;
}
/* 4. process frame irq */
if (irq_status & INT_STA_ENC_DONE_STA) {
mpp->irq_status = irq_status;
if (enc->bs_overflow) {
mpp->irq_status |= INT_STA_BSF_OFLW_STA;
enc->bs_overflow = 0;
}
ret = IRQ_WAKE_THREAD;
}
if (irq_mask)
mpp_write(mpp, hw->int_mask_base, irq_mask);
/* 5. process error irq */
if (irq_status & INT_STA_ERROR) {
mpp->irq_status = irq_status;
if (int_clear) {
mpp_write(mpp, hw->int_clr_base, mpp->irq_status);
udelay(5);
mpp_write(mpp, hw->int_sta_base, 0);
dev_err(mpp->dev, "found error status %08x\n", irq_status);
ret = IRQ_WAKE_THREAD;
}
mpp_debug_leave();
@ -1378,6 +1403,11 @@ static int rkvenc_irq(struct mpp_dev *mpp)
return ret;
}
static int vepu540c_irq(struct mpp_dev *mpp)
{
return rkvenc_irq(mpp);
}
static int rkvenc_isr(struct mpp_dev *mpp)
{
struct rkvenc_task *task;
@ -1406,9 +1436,6 @@ static int rkvenc_isr(struct mpp_dev *mpp)
rkvenc2_update_dchs(enc, task);
mpp_debug(DEBUG_IRQ_STATUS, "%s irq_status: %08x\n",
dev_name(mpp->dev), task->irq_status);
if (task->irq_status & enc->hw_info->err_mask) {
atomic_inc(&mpp->reset_request);
@ -1458,7 +1485,7 @@ static int rkvenc_finish(struct mpp_dev *mpp, struct mpp_task *mpp_task)
if (task->bs_buf) {
u32 bs_size = mpp_read(mpp, 0x4064);
mpp_dma_buf_sync(task->bs_buf, 0, bs_size / 8 + task->offset_bs,
mpp_dma_buf_sync(task->bs_buf, 0, bs_size + task->offset_bs,
DMA_FROM_DEVICE, true);
}
@ -1785,16 +1812,19 @@ static int rkvenc_devfreq_init(struct mpp_dev *mpp)
if (IS_ERR(reg_table))
return PTR_ERR(reg_table);
}
enc->opp_table = reg_table;
clk_table = dev_pm_opp_set_clkname(dev, "clk_core");
if (IS_ERR(clk_table))
return PTR_ERR(clk_table);
if (IS_ERR(clk_table)) {
ret = PTR_ERR(clk_table);
goto put_opp_reg;
}
rockchip_get_opp_data(rockchip_rkvenc_of_match, &enc->opp_info);
ret = rockchip_init_opp_table(dev, &enc->opp_info, "leakage", "venc");
if (ret) {
dev_err(dev, "failed to init_opp_table\n");
return ret;
goto put_opp_clk;
}
enc->mdev_info = rockchip_system_monitor_register(dev, &venc_mdevp);
@ -1803,6 +1833,14 @@ static int rkvenc_devfreq_init(struct mpp_dev *mpp)
enc->mdev_info = NULL;
}
return 0;
put_opp_clk:
dev_pm_opp_put_clkname(enc->opp_table);
put_opp_reg:
dev_pm_opp_put_regulators(enc->opp_table);
enc->opp_table = NULL;
return ret;
}
@ -1810,8 +1848,16 @@ static int rkvenc_devfreq_remove(struct mpp_dev *mpp)
{
struct rkvenc_dev *enc = to_rkvenc_dev(mpp);
if (enc->mdev_info)
if (enc->mdev_info) {
rockchip_system_monitor_unregister(enc->mdev_info);
enc->mdev_info = NULL;
}
if (enc->opp_table) {
rockchip_uninit_opp_table(mpp->dev, &enc->opp_info);
dev_pm_opp_put_clkname(enc->opp_table);
dev_pm_opp_put_regulators(enc->opp_table);
enc->opp_table = NULL;
}
return 0;
}
@ -1880,7 +1926,7 @@ static int rkvenc_soft_reset(struct mpp_dev *mpp)
/* safe reset */
mpp_write(mpp, hw->int_mask_base, 0x3FF);
mpp_write(mpp, hw->enc_clr_base, 0x1);
mpp_write(mpp, hw->enc_clr_base, 0x3);
ret = readl_relaxed_poll_timeout(mpp->reg_base + hw->int_sta_base,
rst_status,
rst_status & RKVENC_SCLR_DONE_STA,
@ -2169,6 +2215,20 @@ static struct mpp_dev_ops rkvenc_ccu_dev_ops = {
.dump_session = rkvenc_dump_session,
};
static struct mpp_dev_ops vepu540c_dev_ops_v2 = {
.wait_result = rkvenc2_wait_result,
.alloc_task = rkvenc_alloc_task,
.run = rkvenc_run,
.irq = vepu540c_irq,
.isr = rkvenc_isr,
.finish = rkvenc_finish,
.result = rkvenc_result,
.free_task = rkvenc_free_task,
.ioctl = rkvenc_control,
.init_session = rkvenc_init_session,
.free_session = rkvenc_free_session,
.dump_session = rkvenc_dump_session,
};
static const struct mpp_dev_var rkvenc_v2_data = {
.device_type = MPP_DEVICE_RKVENC,
@ -2183,7 +2243,7 @@ static const struct mpp_dev_var rkvenc_540c_data = {
.hw_info = &rkvenc_540c_hw_info.hw,
.trans_info = trans_rkvenc_540c,
.hw_ops = &rkvenc_hw_ops,
.dev_ops = &rkvenc_dev_ops_v2,
.dev_ops = &vepu540c_dev_ops_v2,
};
static const struct mpp_dev_var rkvenc_ccu_data = {
@ -2404,14 +2464,33 @@ static int rkvenc2_iommu_fault_handle(struct iommu_domain *iommu,
{
struct mpp_dev *mpp = (struct mpp_dev *)arg;
struct rkvenc_dev *enc = to_rkvenc_dev(mpp);
struct mpp_task *mpp_task = mpp->cur_task;
struct mpp_task *mpp_task;
struct rkvenc_ccu *ccu = enc->ccu;
if (ccu) {
struct rkvenc_dev *core = NULL, *n;
list_for_each_entry_safe(core, n, &ccu->core_list, core_link) {
if (core->mpp.iommu_info &&
(&core->mpp.iommu_info->pdev->dev == iommu_dev)) {
mpp = &core->mpp;
break;
}
}
}
mpp_task = mpp->cur_task;
dev_info(mpp->dev, "core %d page fault found dchs %08x\n",
mpp->core_id, mpp_read_relaxed(&enc->mpp, DCHS_REG_OFFSET));
if (mpp_task)
mpp_task_dump_mem_region(mpp, mpp_task);
/*
* Mask iommu irq, in order for iommu not repeatedly trigger pagefault.
* Until the pagefault task finish by hw timeout.
*/
rockchip_iommu_mask_irq(mpp->dev);
return 0;
}
@ -2455,7 +2534,7 @@ static int rkvenc_core_probe(struct platform_device *pdev)
ret = devm_request_threaded_irq(dev, mpp->irq,
mpp_dev_irq,
mpp_dev_isr_sched,
IRQF_SHARED,
IRQF_ONESHOT,
dev_name(dev), mpp);
if (ret) {
dev_err(dev, "register interrupter runtime failed\n");

29
include/linux/rockchip/rockchip_sip.h
View File

@ -57,6 +57,7 @@
#define SIP_WDT_CFG 0x82000026
#define SIP_HDMIRX_CFG 0x82000027
#define SIP_MCU_CFG 0x82000028
#define SIP_PVTPLL_CFG 0x82000029
#define TRUSTED_OS_HDCPKEY_INIT 0xB7000003
@ -226,6 +227,13 @@ enum {
HDMIRX_INFO_NOTIFY = 2,
};
/* SIP_PVTPLL_CFG child configs */
enum {
PVTPLL_GET_INFO = 0,
PVTPLL_ADJUST_TABLE = 1,
PVTPLL_LOW_TEMP = 2,
};
struct pt_regs;
typedef void (*sip_fiq_debugger_uart_irq_tf_cb_t)(struct pt_regs *_pt_regs, unsigned long cpu);
@ -256,6 +264,9 @@ struct arm_smccc_res sip_smc_bus_config(u32 arg0, u32 arg1, u32 arg2);
struct dram_addrmap_info *sip_smc_get_dram_map(void);
int sip_smc_amp_config(u32 sub_func_id, u32 arg1, u32 arg2, u32 arg3);
struct arm_smccc_res sip_smc_get_amp_info(u32 sub_func_id, u32 arg1);
struct arm_smccc_res sip_smc_get_pvtpll_info(u32 sub_func_id, u32 arg1);
struct arm_smccc_res sip_smc_pvtpll_config(u32 sub_func_id, u32 arg1, u32 arg2,
u32 arg3, u32 arg4, u32 arg5, u32 arg6);
void __iomem *sip_hdcp_request_share_memory(int id);
struct arm_smccc_res sip_hdcp_config(u32 arg0, u32 arg1, u32 arg2);
@ -365,6 +376,24 @@ static inline struct arm_smccc_res sip_smc_get_amp_info(u32 sub_func_id,
return tmp;
}
static inline struct arm_smccc_res sip_smc_get_pvtpll_info(u32 sub_func_id,
u32 arg1)
{
struct arm_smccc_res tmp = { .a0 = SIP_RET_NOT_SUPPORTED, };
return tmp;
}
static inline struct arm_smccc_res sip_smc_pvtpll_config(u32 sub_func_id,
u32 arg1, u32 arg2,
u32 arg3, u32 arg4,
u32 arg5, u32 arg6)
{
struct arm_smccc_res tmp = { .a0 = SIP_RET_NOT_SUPPORTED, };
return tmp;
}
static inline void __iomem *sip_hdcp_request_share_memory(int id)
{
return NULL;

15
include/soc/rockchip/rockchip_opp_select.h
View File

@ -78,6 +78,8 @@ struct rockchip_opp_info {
u32 low_rm;
u32 current_rm;
u32 target_rm;
u32 pvtpll_clk_id;
bool pvtpll_low_temp;
};
#if IS_ENABLED(CONFIG_ROCKCHIP_OPP)
@ -87,6 +89,7 @@ void rockchip_of_get_lkg_sel(struct device *dev, struct device_node *np,
int *volt_sel, int *scale_sel);
void rockchip_pvtpll_calibrate_opp(struct rockchip_opp_info *info);
void rockchip_pvtpll_add_length(struct rockchip_opp_info *info);
void rockchip_init_pvtpll_table(struct rockchip_opp_info *info, int bin);
void rockchip_of_get_pvtm_sel(struct device *dev, struct device_node *np,
char *reg_name, int bin, int process,
int *volt_sel, int *scale_sel);
@ -130,6 +133,8 @@ int rockchip_set_intermediate_rate(struct device *dev,
int rockchip_init_opp_table(struct device *dev,
struct rockchip_opp_info *info,
char *lkg_name, char *reg_name);
void rockchip_uninit_opp_table(struct device *dev,
struct rockchip_opp_info *info);
#else
static inline int rockchip_of_get_leakage(struct device *dev, char *lkg_name,
int *leakage)
@ -152,6 +157,11 @@ static inline void rockchip_pvtpll_add_length(struct rockchip_opp_info *info)
{
}
static inline void rockchip_init_pvtpll_table(struct rockchip_opp_info *info,
int bin)
{
}
static inline void rockchip_of_get_pvtm_sel(struct device *dev,
struct device_node *np,
char *reg_name, int bin, int process,
@ -266,6 +276,11 @@ static inline int rockchip_init_opp_table(struct device *dev,
return -EOPNOTSUPP;
}
static inline void rockchip_uninit_opp_table(struct device *dev,
struct rockchip_opp_info *info)
{
}
#endif /* CONFIG_ROCKCHIP_OPP */
#endif