diff options
193 files changed, 5404 insertions, 1141 deletions
diff --git a/Documentation/devicetree/bindings/mtd/amlogic,meson-nand.txt b/Documentation/devicetree/bindings/mtd/amlogic,meson-nand.txt new file mode 100644 index 000000000000..3983c11e062c --- /dev/null +++ b/Documentation/devicetree/bindings/mtd/amlogic,meson-nand.txt @@ -0,0 +1,60 @@ +Amlogic NAND Flash Controller (NFC) for GXBB/GXL/AXG family SoCs + +This file documents the properties in addition to those available in +the MTD NAND bindings. + +Required properties: +- compatible : contains one of: + - "amlogic,meson-gxl-nfc" + - "amlogic,meson-axg-nfc" +- clocks : + A list of phandle + clock-specifier pairs for the clocks listed + in clock-names. + +- clock-names: Should contain the following: + "core" - NFC module gate clock + "device" - device clock from eMMC sub clock controller + "rx" - rx clock phase + "tx" - tx clock phase + +- amlogic,mmc-syscon : Required for NAND clocks, it's shared with SD/eMMC + controller port C + +Optional children nodes: +Children nodes represent the available nand chips. + +Other properties: +see Documentation/devicetree/bindings/mtd/nand.txt for generic bindings. + +Example demonstrate on AXG SoC: + + sd_emmc_c_clkc: mmc@7000 { + compatible = "amlogic,meson-axg-mmc-clkc", "syscon"; + reg = <0x0 0x7000 0x0 0x800>; + }; + + nand-controller@7800 { + compatible = "amlogic,meson-axg-nfc"; + reg = <0x0 0x7800 0x0 0x100>; + #address-cells = <1>; + #size-cells = <0>; + interrupts = <GIC_SPI 34 IRQ_TYPE_EDGE_RISING>; + + clocks = <&clkc CLKID_SD_EMMC_C>, + <&sd_emmc_c_clkc CLKID_MMC_DIV>, + <&sd_emmc_c_clkc CLKID_MMC_PHASE_RX>, + <&sd_emmc_c_clkc CLKID_MMC_PHASE_TX>; + clock-names = "core", "device", "rx", "tx"; + amlogic,mmc-syscon = <&sd_emmc_c_clkc>; + + pinctrl-names = "default"; + pinctrl-0 = <&nand_pins>; + + nand@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <1>; + + nand-on-flash-bbt; + }; + }; diff --git a/Documentation/devicetree/bindings/mtd/cadence-quadspi.txt b/Documentation/devicetree/bindings/mtd/cadence-quadspi.txt index bb2075df9b38..4345c3a6f530 100644 --- a/Documentation/devicetree/bindings/mtd/cadence-quadspi.txt +++ b/Documentation/devicetree/bindings/mtd/cadence-quadspi.txt @@ -4,6 +4,7 @@ Required properties: - compatible : should be one of the following: Generic default - "cdns,qspi-nor". For TI 66AK2G SoC - "ti,k2g-qspi", "cdns,qspi-nor". + For TI AM654 SoC - "ti,am654-ospi", "cdns,qspi-nor". - reg : Contains two entries, each of which is a tuple consisting of a physical address and length. The first entry is the address and length of the controller register set. The second entry is the diff --git a/Documentation/devicetree/bindings/mtd/mtk-quadspi.txt b/Documentation/devicetree/bindings/mtd/mtk-quadspi.txt index 56d3668e2c50..a12e3b5c495d 100644 --- a/Documentation/devicetree/bindings/mtd/mtk-quadspi.txt +++ b/Documentation/devicetree/bindings/mtd/mtk-quadspi.txt @@ -1,4 +1,4 @@ -* Serial NOR flash controller for MTK MT81xx (and similar) +* Serial NOR flash controller for MediaTek SoCs Required properties: - compatible: For mt8173, compatible should be "mediatek,mt8173-nor", @@ -10,6 +10,7 @@ Required properties: "mediatek,mt2712-nor", "mediatek,mt8173-nor" "mediatek,mt7622-nor", "mediatek,mt8173-nor" "mediatek,mt7623-nor", "mediatek,mt8173-nor" + "mediatek,mt7629-nor", "mediatek,mt8173-nor" "mediatek,mt8173-nor" - reg: physical base address and length of the controller's register - clocks: the phandle of the clocks needed by the nor controller diff --git a/Documentation/devicetree/bindings/mtd/stm32-fmc2-nand.txt b/Documentation/devicetree/bindings/mtd/stm32-fmc2-nand.txt new file mode 100644 index 000000000000..ad2bef826582 --- /dev/null +++ b/Documentation/devicetree/bindings/mtd/stm32-fmc2-nand.txt @@ -0,0 +1,61 @@ +STMicroelectronics Flexible Memory Controller 2 (FMC2) +NAND Interface + +Required properties: +- compatible: Should be one of: + * st,stm32mp15-fmc2 +- reg: NAND flash controller memory areas. + First region contains the register location. + Regions 2 to 4 respectively contain the data, command, + and address space for CS0. + Regions 5 to 7 contain the same areas for CS1. +- interrupts: The interrupt number +- pinctrl-0: Standard Pinctrl phandle (see: pinctrl/pinctrl-bindings.txt) +- clocks: The clock needed by the NAND flash controller + +Optional properties: +- resets: Reference to a reset controller asserting the FMC controller +- dmas: DMA specifiers (see: dma/stm32-mdma.txt) +- dma-names: Must be "tx", "rx" and "ecc" + +* NAND device bindings: + +Required properties: +- reg: describes the CS lines assigned to the NAND device. + +Optional properties: +- nand-on-flash-bbt: see nand.txt +- nand-ecc-strength: see nand.txt +- nand-ecc-step-size: see nand.txt + +The following ECC strength and step size are currently supported: + - nand-ecc-strength = <1>, nand-ecc-step-size = <512> (Hamming) + - nand-ecc-strength = <4>, nand-ecc-step-size = <512> (BCH4) + - nand-ecc-strength = <8>, nand-ecc-step-size = <512> (BCH8) (default) + +Example: + + fmc: nand-controller@58002000 { + compatible = "st,stm32mp15-fmc2"; + reg = <0x58002000 0x1000>, + <0x80000000 0x1000>, + <0x88010000 0x1000>, + <0x88020000 0x1000>, + <0x81000000 0x1000>, + <0x89010000 0x1000>, + <0x89020000 0x1000>; + interrupts = <GIC_SPI 48 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&rcc FMC_K>; + resets = <&rcc FMC_R>; + pinctrl-names = "default"; + pinctrl-0 = <&fmc_pins_a>; + #address-cells = <1>; + #size-cells = <0>; + + nand@0 { + reg = <0>; + nand-on-flash-bbt; + #address-cells = <1>; + #size-cells = <1>; + }; + }; diff --git a/MAINTAINERS b/MAINTAINERS index 91eacdbe9cda..84857bc78a61 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -7176,6 +7176,7 @@ F: drivers/i2c/i2c-stub.c I3C SUBSYSTEM M: Boris Brezillon <bbrezillon@kernel.org> L: linux-i3c@lists.infradead.org +C: irc://chat.freenode.net/linux-i3c T: git git://git.kernel.org/pub/scm/linux/kernel/git/i3c/linux.git S: Maintained F: Documentation/ABI/testing/sysfs-bus-i3c @@ -9866,6 +9867,13 @@ F: drivers/media/platform/meson/ao-cec.c F: Documentation/devicetree/bindings/media/meson-ao-cec.txt T: git git://linuxtv.org/media_tree.git +MESON NAND CONTROLLER DRIVER FOR AMLOGIC SOCS +M: Liang Yang <liang.yang@amlogic.com> +L: linux-mtd@lists.infradead.org +S: Maintained +F: drivers/mtd/nand/raw/meson_* +F: Documentation/devicetree/bindings/mtd/amlogic,meson-nand.txt + MICROBLAZE ARCHITECTURE M: Michal Simek <monstr@monstr.eu> W: http://www.monstr.eu/fdt/ @@ -14346,6 +14354,7 @@ F: arch/arm/mach-spear/ SPI NOR SUBSYSTEM M: Marek Vasut <marek.vasut@gmail.com> +M: Tudor Ambarus <tudor.ambarus@microchip.com> L: linux-mtd@lists.infradead.org W: http://www.linux-mtd.infradead.org/ Q: http://patchwork.ozlabs.org/project/linux-mtd/list/ @@ -2,7 +2,7 @@ VERSION = 5 PATCHLEVEL = 0 SUBLEVEL = 0 -EXTRAVERSION = -rc8 +EXTRAVERSION = NAME = Shy Crocodile # *DOCUMENTATION* diff --git a/arch/alpha/include/asm/uaccess.h b/arch/alpha/include/asm/uaccess.h index cf4ac791a592..1fe2b56cb861 100644 --- a/arch/alpha/include/asm/uaccess.h +++ b/arch/alpha/include/asm/uaccess.h @@ -18,7 +18,6 @@ #define USER_DS ((mm_segment_t) { -0x40000000000UL }) #define get_fs() (current_thread_info()->addr_limit) -#define get_ds() (KERNEL_DS) #define set_fs(x) (current_thread_info()->addr_limit = (x)) #define segment_eq(a, b) ((a).seg == (b).seg) diff --git a/arch/arm/boot/dts/gemini-dlink-dir-685.dts b/arch/arm/boot/dts/gemini-dlink-dir-685.dts index cc0c3cf89eaa..592111c8d6fd 100644 --- a/arch/arm/boot/dts/gemini-dlink-dir-685.dts +++ b/arch/arm/boot/dts/gemini-dlink-dir-685.dts @@ -443,7 +443,7 @@ }; display-controller@6a000000 { - status = "disabled"; + status = "okay"; port@0 { reg = <0>; diff --git a/arch/arm/crypto/sha256-armv4.pl b/arch/arm/crypto/sha256-armv4.pl index b9ec44060ed3..a03cf4dfb781 100644 --- a/arch/arm/crypto/sha256-armv4.pl +++ b/arch/arm/crypto/sha256-armv4.pl @@ -212,10 +212,11 @@ K256: .global sha256_block_data_order .type sha256_block_data_order,%function sha256_block_data_order: +.Lsha256_block_data_order: #if __ARM_ARCH__<7 sub r3,pc,#8 @ sha256_block_data_order #else - adr r3,sha256_block_data_order + adr r3,.Lsha256_block_data_order #endif #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) ldr r12,.LOPENSSL_armcap diff --git a/arch/arm/crypto/sha256-core.S_shipped b/arch/arm/crypto/sha256-core.S_shipped index 3b58300d611c..054aae0edfce 100644 --- a/arch/arm/crypto/sha256-core.S_shipped +++ b/arch/arm/crypto/sha256-core.S_shipped @@ -93,10 +93,11 @@ K256: .global sha256_block_data_order .type sha256_block_data_order,%function sha256_block_data_order: +.Lsha256_block_data_order: #if __ARM_ARCH__<7 sub r3,pc,#8 @ sha256_block_data_order #else - adr r3,sha256_block_data_order + adr r3,.Lsha256_block_data_order #endif #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) ldr r12,.LOPENSSL_armcap diff --git a/arch/arm/crypto/sha512-armv4.pl b/arch/arm/crypto/sha512-armv4.pl index fb5d15048c0b..788c17b56ecc 100644 --- a/arch/arm/crypto/sha512-armv4.pl +++ b/arch/arm/crypto/sha512-armv4.pl @@ -274,10 +274,11 @@ WORD64(0x5fcb6fab,0x3ad6faec, 0x6c44198c,0x4a475817) .global sha512_block_data_order .type sha512_block_data_order,%function sha512_block_data_order: +.Lsha512_block_data_order: #if __ARM_ARCH__<7 sub r3,pc,#8 @ sha512_block_data_order #else - adr r3,sha512_block_data_order + adr r3,.Lsha512_block_data_order #endif #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) ldr r12,.LOPENSSL_armcap diff --git a/arch/arm/crypto/sha512-core.S_shipped b/arch/arm/crypto/sha512-core.S_shipped index b1c334a49cda..710ea309769e 100644 --- a/arch/arm/crypto/sha512-core.S_shipped +++ b/arch/arm/crypto/sha512-core.S_shipped @@ -141,10 +141,11 @@ WORD64(0x5fcb6fab,0x3ad6faec, 0x6c44198c,0x4a475817) .global sha512_block_data_order .type sha512_block_data_order,%function sha512_block_data_order: +.Lsha512_block_data_order: #if __ARM_ARCH__<7 sub r3,pc,#8 @ sha512_block_data_order #else - adr r3,sha512_block_data_order + adr r3,.Lsha512_block_data_order #endif #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) ldr r12,.LOPENSSL_armcap diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h index 42aa4a22803c..ae5a0df5316e 100644 --- a/arch/arm/include/asm/uaccess.h +++ b/arch/arm/include/asm/uaccess.h @@ -59,7 +59,6 @@ extern int __put_user_bad(void); * Note that this is actually 0x1,0000,0000 */ #define KERNEL_DS 0x00000000 -#define get_ds() (KERNEL_DS) #ifdef CONFIG_MMU diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index 16601d1442d1..72cc0862a30e 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -150,7 +150,7 @@ void __show_regs(struct pt_regs *regs) if ((domain & domain_mask(DOMAIN_USER)) == domain_val(DOMAIN_USER, DOMAIN_NOACCESS)) segment = "none"; - else if (fs == get_ds()) + else if (fs == KERNEL_DS) segment = "kernel"; else segment = "user"; diff --git a/arch/arm/xen/hypercall.S b/arch/arm/xen/hypercall.S index b0b80c0f09f3..b11bba542fac 100644 --- a/arch/arm/xen/hypercall.S +++ b/arch/arm/xen/hypercall.S @@ -113,8 +113,7 @@ ENTRY(privcmd_call) /* * Disable userspace access from kernel. This is fine to do it - * unconditionally as no set_fs(KERNEL_DS)/set_fs(get_ds()) is - * called before. + * unconditionally as no set_fs(KERNEL_DS) is called before. */ uaccess_disable r4 diff --git a/arch/arm64/boot/dts/qcom/msm8998.dtsi b/arch/arm64/boot/dts/qcom/msm8998.dtsi index 8d41b69ec2da..99bccaac31ad 100644 --- a/arch/arm64/boot/dts/qcom/msm8998.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8998.dtsi @@ -37,7 +37,7 @@ }; memory@86200000 { - reg = <0x0 0x86200000 0x0 0x2600000>; + reg = <0x0 0x86200000 0x0 0x2d00000>; no-map; }; diff --git a/arch/arm64/crypto/chacha-neon-core.S b/arch/arm64/crypto/chacha-neon-core.S index 021bb9e9784b..706c4e10e9e2 100644 --- a/arch/arm64/crypto/chacha-neon-core.S +++ b/arch/arm64/crypto/chacha-neon-core.S @@ -158,8 +158,8 @@ ENTRY(hchacha_block_neon) mov w3, w2 bl chacha_permute - st1 {v0.16b}, [x1], #16 - st1 {v3.16b}, [x1] + st1 {v0.4s}, [x1], #16 + st1 {v3.4s}, [x1] ldp x29, x30, [sp], #16 ret @@ -532,6 +532,10 @@ ENTRY(chacha_4block_xor_neon) add v3.4s, v3.4s, v19.4s add a2, a2, w8 add a3, a3, w9 +CPU_BE( rev a0, a0 ) +CPU_BE( rev a1, a1 ) +CPU_BE( rev a2, a2 ) +CPU_BE( rev a3, a3 ) ld4r {v24.4s-v27.4s}, [x0], #16 ld4r {v28.4s-v31.4s}, [x0] @@ -552,6 +556,10 @@ ENTRY(chacha_4block_xor_neon) add v7.4s, v7.4s, v23.4s add a6, a6, w8 add a7, a7, w9 +CPU_BE( rev a4, a4 ) +CPU_BE( rev a5, a5 ) +CPU_BE( rev a6, a6 ) +CPU_BE( rev a7, a7 ) // x8[0-3] += s2[0] // x9[0-3] += s2[1] @@ -569,6 +577,10 @@ ENTRY(chacha_4block_xor_neon) add v11.4s, v11.4s, v27.4s add a10, a10, w8 add a11, a11, w9 +CPU_BE( rev a8, a8 ) +CPU_BE( rev a9, a9 ) +CPU_BE( rev a10, a10 ) +CPU_BE( rev a11, a11 ) // x12[0-3] += s3[0] // x13[0-3] += s3[1] @@ -586,6 +598,10 @@ ENTRY(chacha_4block_xor_neon) add v15.4s, v15.4s, v31.4s add a14, a14, w8 add a15, a15, w9 +CPU_BE( rev a12, a12 ) +CPU_BE( rev a13, a13 ) +CPU_BE( rev a14, a14 ) +CPU_BE( rev a15, a15 ) // interleave 32-bit words in state n, n+1 ldp w6, w7, [x2], #64 diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index 547d7a0c9d05..f1e5c9165809 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -34,7 +34,6 @@ #include <asm/memory.h> #include <asm/extable.h> -#define get_ds() (KERNEL_DS) #define get_fs() (current_thread_info()->addr_limit) static inline void set_fs(mm_segment_t fs) diff --git a/arch/csky/include/asm/segment.h b/arch/csky/include/asm/segment.h index ffdc4c47ff43..db2640d5f575 100644 --- a/arch/csky/include/asm/segment.h +++ b/arch/csky/include/asm/segment.h @@ -9,7 +9,6 @@ typedef struct { } mm_segment_t; #define KERNEL_DS ((mm_segment_t) { 0xFFFFFFFF }) -#define get_ds() KERNEL_DS #define USER_DS ((mm_segment_t) { 0x80000000UL }) #define get_fs() (current_thread_info()->addr_limit) diff --git a/arch/h8300/include/asm/segment.h b/arch/h8300/include/asm/segment.h index 9adbf7e1aaa3..a407978f9f9f 100644 --- a/arch/h8300/include/asm/segment.h +++ b/arch/h8300/include/asm/segment.h @@ -33,12 +33,6 @@ static inline mm_segment_t get_fs(void) return USER_DS; } -static inline mm_segment_t get_ds(void) -{ - /* return the supervisor data space code */ - return KERNEL_DS; -} - #define segment_eq(a, b) ((a).seg == (b).seg) #endif /* __ASSEMBLY__ */ diff --git a/arch/ia64/include/asm/uaccess.h b/arch/ia64/include/asm/uaccess.h index 306d469e43da..89782ad3fb88 100644 --- a/arch/ia64/include/asm/uaccess.h +++ b/arch/ia64/include/asm/uaccess.h @@ -48,7 +48,6 @@ #define KERNEL_DS ((mm_segment_t) { ~0UL }) /* cf. access_ok() */ #define USER_DS ((mm_segment_t) { TASK_SIZE-1 }) /* cf. access_ok() */ -#define get_ds() (KERNEL_DS) #define get_fs() (current_thread_info()->addr_limit) #define set_fs(x) (current_thread_info()->addr_limit = (x)) diff --git a/arch/m68k/include/asm/segment.h b/arch/m68k/include/asm/segment.h index 0b4cc1e079b5..c6686559e9b7 100644 --- a/arch/m68k/include/asm/segment.h +++ b/arch/m68k/include/asm/segment.h @@ -45,16 +45,9 @@ static inline void set_fs(mm_segment_t val) : /* no outputs */ : "r" (val.seg) : "memory"); } -static inline mm_segment_t get_ds(void) -{ - /* return the supervisor data space code */ - return KERNEL_DS; -} - #else #define USER_DS MAKE_MM_SEG(TASK_SIZE) #define KERNEL_DS MAKE_MM_SEG(0xFFFFFFFF) -#define get_ds() (KERNEL_DS) #define get_fs() (current_thread_info()->addr_limit) #define set_fs(x) (current_thread_info()->addr_limit = (x)) #endif diff --git a/arch/microblaze/include/asm/uaccess.h b/arch/microblaze/include/asm/uaccess.h index dbfea093a7c7..bff2a71c828a 100644 --- a/arch/microblaze/include/asm/uaccess.h +++ b/arch/microblaze/include/asm/uaccess.h @@ -42,7 +42,6 @@ # define USER_DS MAKE_MM_SEG(TASK_SIZE - 1) # endif -# define get_ds() (KERNEL_DS) # define get_fs() (current_thread_info()->addr_limit) # define set_fs(val) (current_thread_info()->addr_limit = (val)) diff --git a/arch/mips/bcm63xx/dev-enet.c b/arch/mips/bcm63xx/dev-enet.c index 07b4c65a88a4..8e73d65f3480 100644 --- a/arch/mips/bcm63xx/dev-enet.c +++ b/arch/mips/bcm63xx/dev-enet.c @@ -70,6 +70,8 @@ static struct platform_device bcm63xx_enet_shared_device = { static int shared_device_registered; +static u64 enet_dmamask = DMA_BIT_MASK(32); + static struct resource enet0_res[] = { { .start = -1, /* filled at runtime */ @@ -99,6 +101,8 @@ static struct platform_device bcm63xx_enet0_device = { .resource = enet0_res, .dev = { .platform_data = &enet0_pd, + .dma_mask = &enet_dmamask, + .coherent_dma_mask = DMA_BIT_MASK(32), }, }; @@ -131,6 +135,8 @@ static struct platform_device bcm63xx_enet1_device = { .resource = enet1_res, .dev = { .platform_data = &enet1_pd, + .dma_mask = &enet_dmamask, + .coherent_dma_mask = DMA_BIT_MASK(32), }, }; @@ -157,6 +163,8 @@ static struct platform_device bcm63xx_enetsw_device = { .resource = enetsw_res, .dev = { .platform_data = &enetsw_pd, + .dma_mask = &enet_dmamask, + .coherent_dma_mask = DMA_BIT_MASK(32), }, }; diff --git a/arch/mips/include/asm/uaccess.h b/arch/mips/include/asm/uaccess.h index d43c1dc6ef15..62b298c50905 100644 --- a/arch/mips/include/asm/uaccess.h +++ b/arch/mips/include/asm/uaccess.h @@ -69,7 +69,6 @@ extern u64 __ua_limit; #define USER_DS ((mm_segment_t) { __UA_LIMIT }) #endif -#define get_ds() (KERNEL_DS) #define get_fs() (current_thread_info()->addr_limit) #define set_fs(x) (current_thread_info()->addr_limit = (x)) diff --git a/arch/mips/kernel/cmpxchg.c b/arch/mips/kernel/cmpxchg.c index 0b9535bc2c53..6b2a4a902a98 100644 --- a/arch/mips/kernel/cmpxchg.c +++ b/arch/mips/kernel/cmpxchg.c @@ -54,10 +54,9 @@ unsigned long __xchg_small(volatile void *ptr, unsigned long val, unsigned int s unsigned long __cmpxchg_small(volatile void *ptr, unsigned long old, unsigned long new, unsigned int size) { - u32 mask, old32, new32, load32; + u32 mask, old32, new32, load32, load; volatile u32 *ptr32; unsigned int shift; - u8 load; /* Check that ptr is naturally aligned */ WARN_ON((unsigned long)ptr & (size - 1)); diff --git a/arch/mips/kernel/ftrace.c b/arch/mips/kernel/ftrace.c index 2ea0ec95efe9..4b5e1f2bfbce 100644 --- a/arch/mips/kernel/ftrace.c +++ b/arch/mips/kernel/ftrace.c @@ -86,7 +86,7 @@ static int ftrace_modify_code(unsigned long ip, unsigned int new_code) return -EFAULT; old_fs = get_fs(); - set_fs(get_ds()); + set_fs(KERNEL_DS); flush_icache_range(ip, ip + 8); set_fs(old_fs); @@ -111,7 +111,7 @@ static int ftrace_modify_code_2(unsigned long ip, unsigned int new_code1, ip -= 4; old_fs = get_fs(); - set_fs(get_ds()); + set_fs(KERNEL_DS); flush_icache_range(ip, ip + 8); set_fs(old_fs); @@ -135,7 +135,7 @@ static int ftrace_modify_code_2r(unsigned long ip, unsigned int new_code1, return -EFAULT; old_fs = get_fs(); - set_fs(get_ds()); + set_fs(KERNEL_DS); flush_icache_range(ip, ip + 8); set_fs(old_fs); diff --git a/arch/mips/kernel/kgdb.c b/arch/mips/kernel/kgdb.c index 149100e1bc7c..6e574c02e4c3 100644 --- a/arch/mips/kernel/kgdb.c +++ b/arch/mips/kernel/kgdb.c @@ -212,7 +212,7 @@ void kgdb_call_nmi_hook(void *ignored) mm_segment_t old_fs; old_fs = get_fs(); - set_fs(get_ds()); + set_fs(KERNEL_DS); kgdb_nmicallback(raw_smp_processor_id(), NULL); @@ -318,7 +318,7 @@ static int kgdb_mips_notify(struct notifier_block *self, unsigned long cmd, /* Kernel mode. Set correct address limit */ old_fs = get_fs(); - set_fs(get_ds()); + set_fs(KERNEL_DS); if (atomic_read(&kgdb_active) != -1) kgdb_nmicallback(smp_processor_id(), regs); diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c index 8c6c48ed786a..d2e5a5ad0e6f 100644 --- a/arch/mips/kernel/setup.c +++ b/arch/mips/kernel/setup.c @@ -384,7 +384,8 @@ static void __init bootmem_init(void) init_initrd(); reserved_end = (unsigned long) PFN_UP(__pa_symbol(&_end)); - memblock_reserve(PHYS_OFFSET, reserved_end << PAGE_SHIFT); + memblock_reserve(PHYS_OFFSET, + (reserved_end << PAGE_SHIFT) - PHYS_OFFSET); /* * max_low_pfn is not a number of pages. The number of pages diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c index c91097f7b32f..cbab46004e99 100644 --- a/arch/mips/kernel/traps.c +++ b/arch/mips/kernel/traps.c @@ -1077,7 +1077,7 @@ asmlinkage void do_tr(struct pt_regs *regs) seg = get_fs(); if (!user_mode(regs)) - set_fs(get_ds()); + set_fs(KERNEL_DS); prev_state = exception_enter(); current->thread.trap_nr = (regs->cp0_cause >> 2) & 0x1f; diff --git a/arch/mips/lantiq/xway/vmmc.c b/arch/mips/lantiq/xway/vmmc.c index 577ec81b557d..3deab9a77718 100644 --- a/arch/mips/lantiq/xway/vmmc.c +++ b/arch/mips/lantiq/xway/vmmc.c @@ -31,8 +31,8 @@ static int vmmc_probe(struct platform_device *pdev) dma_addr_t dma; cp1_base = - (void *) CPHYSADDR(dma_alloc_coherent(NULL, CP1_SIZE, - &dma, GFP_ATOMIC)); + (void *) CPHYSADDR(dma_alloc_coherent(&pdev->dev, CP1_SIZE, + &dma, GFP_KERNEL)); gpio_count = of_gpio_count(pdev->dev.of_node); while (gpio_count > 0) { diff --git a/arch/mips/net/ebpf_jit.c b/arch/mips/net/ebpf_jit.c index 76e9bf88d3b9..0effd3cba9a7 100644 --- a/arch/mips/net/ebpf_jit.c +++ b/arch/mips/net/ebpf_jit.c @@ -1819,7 +1819,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) /* Update the icache */ flush_icache_range((unsigned long)ctx.target, - (unsigned long)(ctx.target + ctx.idx * sizeof(u32))); + (unsigned long)&ctx.target[ctx.idx]); if (bpf_jit_enable > 1) /* Dump JIT code */ diff --git a/arch/nds32/include/asm/uaccess.h b/arch/nds32/include/asm/uaccess.h index 53dcb49b0b12..116598b47c4d 100644 --- a/arch/nds32/include/asm/uaccess.h +++ b/arch/nds32/include/asm/uaccess.h @@ -37,7 +37,6 @@ extern int fixup_exception(struct pt_regs *regs); #define KERNEL_DS ((mm_segment_t) { ~0UL }) #define USER_DS ((mm_segment_t) {TASK_SIZE - 1}) -#define get_ds() (KERNEL_DS) #define get_fs() (current_thread_info()->addr_limit) #define user_addr_max get_fs diff --git a/arch/nds32/kernel/process.c b/arch/nds32/kernel/process.c index ab7ab46234b1..9712fd474f2c 100644 --- a/arch/nds32/kernel/process.c +++ b/arch/nds32/kernel/process.c @@ -121,7 +121,7 @@ void show_regs(struct pt_regs *regs) regs->uregs[3], regs->uregs[2], regs->uregs[1], regs->uregs[0]); pr_info(" IRQs o%s Segment %s\n", interrupts_enabled(regs) ? "n" : "ff", - segment_eq(get_fs(), get_ds())? "kernel" : "user"); + segment_eq(get_fs(), KERNEL_DS)? "kernel" : "user"); } EXPORT_SYMBOL(show_regs); diff --git a/arch/nios2/include/asm/uaccess.h b/arch/nios2/include/asm/uaccess.h index e0ea10806491..e83f831a76f9 100644 --- a/arch/nios2/include/asm/uaccess.h +++ b/arch/nios2/include/asm/uaccess.h @@ -26,7 +26,6 @@ #define USER_DS MAKE_MM_SEG(0x80000000UL) #define KERNEL_DS MAKE_MM_SEG(0) -#define get_ds() (KERNEL_DS) #define get_fs() (current_thread_info()->addr_limit) #define set_fs(seg) (current_thread_info()->addr_limit = (seg)) diff --git a/arch/openrisc/include/asm/uaccess.h b/arch/openrisc/include/asm/uaccess.h index a44682c8adc3..45afd9ab78c1 100644 --- a/arch/openrisc/include/asm/uaccess.h +++ b/arch/openrisc/include/asm/uaccess.h @@ -42,7 +42,6 @@ */ #define KERNEL_DS (~0UL) -#define get_ds() (KERNEL_DS) #define USER_DS (TASK_SIZE) #define get_fs() (current_thread_info()->addr_limit) diff --git a/arch/parisc/include/asm/uaccess.h b/arch/parisc/include/asm/uaccess.h index 30ac2865ea73..ebbb9ffe038c 100644 --- a/arch/parisc/include/asm/uaccess.h +++ b/arch/parisc/include/asm/uaccess.h @@ -16,7 +16,6 @@ #define segment_eq(a, b) ((a).seg == (b).seg) -#define get_ds() (KERNEL_DS) #define get_fs() (current_thread_info()->addr_limit) #define set_fs(x) (current_thread_info()->addr_limit = (x)) diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h index e3a731793ea2..4d6d905e9138 100644 --- a/arch/powerpc/include/asm/uaccess.h +++ b/arch/powerpc/include/asm/uaccess.h @@ -28,7 +28,6 @@ #define USER_DS MAKE_MM_SEG(TASK_SIZE - 1) #endif -#define get_ds() (KERNEL_DS) #define get_fs() (current->thread.addr_limit) static inline void set_fs(mm_segment_t fs) diff --git a/arch/riscv/include/asm/uaccess.h b/arch/riscv/include/asm/uaccess.h index 637b896894fc..a00168b980d2 100644 --- a/arch/riscv/include/asm/uaccess.h +++ b/arch/riscv/include/asm/uaccess.h @@ -41,7 +41,6 @@ #define KERNEL_DS (~0UL) #define USER_DS (TASK_SIZE) -#define get_ds() (KERNEL_DS) #define get_fs() (current_thread_info()->addr_limit) static inline void set_fs(mm_segment_t fs) diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index bd2545977ad3..007fcb9aeeb8 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -31,7 +31,6 @@ #define USER_DS (2) #define USER_DS_SACF (3) -#define get_ds() (KERNEL_DS) #define get_fs() (current->thread.mm_segment) #define segment_eq(a,b) (((a) & 2) == ((b) & 2)) diff --git a/arch/sh/include/asm/segment.h b/arch/sh/include/asm/segment.h index 101c13c0c6ad..33d1d28057cb 100644 --- a/arch/sh/include/asm/segment.h +++ b/arch/sh/include/asm/segment.h @@ -26,7 +26,6 @@ typedef struct { #define segment_eq(a, b) ((a).seg == (b).seg) -#define get_ds() (KERNEL_DS) #define get_fs() (current_thread_info()->addr_limit) #define set_fs(x) (current_thread_info()->addr_limit = (x)) diff --git a/arch/sparc/include/asm/uaccess_32.h b/arch/sparc/include/asm/uaccess_32.h index 5153798051fb..d6d8413eca83 100644 --- a/arch/sparc/include/asm/uaccess_32.h +++ b/arch/sparc/include/asm/uaccess_32.h @@ -25,7 +25,6 @@ #define KERNEL_DS ((mm_segment_t) { 0 }) #define USER_DS ((mm_segment_t) { -1 }) -#define get_ds() (KERNEL_DS) #define get_fs() (current->thread.current_ds) #define set_fs(val) ((current->thread.current_ds) = (val)) diff --git a/arch/sparc/include/asm/uaccess_64.h b/arch/sparc/include/asm/uaccess_64.h index 87ae9ffb1521..bf9d330073b2 100644 --- a/arch/sparc/include/asm/uaccess_64.h +++ b/arch/sparc/include/asm/uaccess_64.h @@ -31,7 +31,6 @@ #define USER_DS ((mm_segment_t) { ASI_AIUS }) /* har har har */ #define get_fs() ((mm_segment_t){(current_thread_info()->current_ds)}) -#define get_ds() (KERNEL_DS) #define segment_eq(a, b) ((a).seg == (b).seg) diff --git a/arch/x86/include/asm/hyperv-tlfs.h b/arch/x86/include/asm/hyperv-tlfs.h index 705dafc2d11a..2bdbbbcfa393 100644 --- a/arch/x86/include/asm/hyperv-tlfs.h +++ b/arch/x86/include/asm/hyperv-tlfs.h @@ -841,7 +841,7 @@ union hv_gpa_page_range { * count is equal with how many entries of union hv_gpa_page_range can * be populated into the input parameter page. */ -#define HV_MAX_FLUSH_REP_COUNT (PAGE_SIZE - 2 * sizeof(u64) / \ +#define HV_MAX_FLUSH_REP_COUNT ((PAGE_SIZE - 2 * sizeof(u64)) / \ sizeof(union hv_gpa_page_range)) struct hv_guest_mapping_flush_list { diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h index 780f2b42c8ef..5e49a0acb5ee 100644 --- a/arch/x86/include/asm/uaccess.h +++ b/arch/x86/include/asm/uaccess.h @@ -25,7 +25,6 @@ #define KERNEL_DS MAKE_MM_SEG(-1UL) #define USER_DS MAKE_MM_SEG(TASK_SIZE_MAX) -#define get_ds() (KERNEL_DS) #define get_fs() (current->thread.addr_limit) static inline void set_fs(mm_segment_t fs) { @@ -284,7 +283,7 @@ do { \ __put_user_goto(x, ptr, "l", "k", "ir", label); \ break; \ case 8: \ - __put_user_goto_u64((__typeof__(*ptr))(x), ptr, label); \ + __put_user_goto_u64(x, ptr, label); \ break; \ default: \ __put_user_bad(); \ @@ -431,8 +430,10 @@ do { \ ({ \ __label__ __pu_label; \ int __pu_err = -EFAULT; \ + __typeof__(*(ptr)) __pu_val; \ + __pu_val = x; \ __uaccess_begin(); \ - __put_user_size((x), (ptr), (size), __pu_label); \ + __put_user_size(__pu_val, (ptr), (size), __pu_label); \ __pu_err = 0; \ __pu_label: \ __uaccess_end(); \ diff --git a/arch/x86/mm/extable.c b/arch/x86/mm/extable.c index 6521134057e8..3c4568f8fb28 100644 --- a/arch/x86/mm/extable.c +++ b/arch/x86/mm/extable.c @@ -117,67 +117,12 @@ __visible bool ex_handler_fprestore(const struct exception_table_entry *fixup, } EXPORT_SYMBOL_GPL(ex_handler_fprestore); -/* Helper to check whether a uaccess fault indicates a kernel bug. */ -static bool bogus_uaccess(struct pt_regs *regs, int trapnr, - unsigned long fault_addr) -{ - /* This is the normal case: #PF with a fault address in userspace. */ - if (trapnr == X86_TRAP_PF && fault_addr < TASK_SIZE_MAX) - return false; - - /* - * This code can be reached for machine checks, but only if the #MC - * handler has already decided that it looks like a candidate for fixup. - * This e.g. happens when attempting to access userspace memory which - * the CPU can't access because of uncorrectable bad memory. - */ - if (trapnr == X86_TRAP_MC) - return false; - - /* - * There are two remaining exception types we might encounter here: - * - #PF for faulting accesses to kernel addresses - * - #GP for faulting accesses to noncanonical addresses - * Complain about anything else. - */ - if (trapnr != X86_TRAP_PF && trapnr != X86_TRAP_GP) { - WARN(1, "unexpected trap %d in uaccess\n", trapnr); - return false; - } - - /* - * This is a faulting memory access in kernel space, on a kernel - * address, in a usercopy function. This can e.g. be caused by improper - * use of helpers like __put_user and by improper attempts to access - * userspace addresses in KERNEL_DS regions. - * The one (semi-)legitimate exception are probe_kernel_{read,write}(), - * which can be invoked from places like kgdb, /dev/mem (for reading) - * and privileged BPF code (for reading). - * The probe_kernel_*() functions set the kernel_uaccess_faults_ok flag - * to tell us that faulting on kernel addresses, and even noncanonical - * addresses, in a userspace accessor does not necessarily imply a - * kernel bug, root might just be doing weird stuff. - */ - if (current->kernel_uaccess_faults_ok) - return false; - - /* This is bad. Refuse the fixup so that we go into die(). */ - if (trapnr == X86_TRAP_PF) { - pr_emerg("BUG: pagefault on kernel address 0x%lx in non-whitelisted uaccess\n", - fault_addr); - } else { - pr_emerg("BUG: GPF in non-whitelisted uaccess (non-canonical address?)\n"); - } - return true; -} - __visible bool ex_handler_uaccess(const struct exception_table_entry *fixup, struct pt_regs *regs, int trapnr, unsigned long error_code, unsigned long fault_addr) { - if (bogus_uaccess(regs, trapnr, fault_addr)) - return false; + WARN_ONCE(trapnr == X86_TRAP_GP, "General protection fault in user access. Non-canonical address?"); regs->ip = ex_fixup_addr(fixup); return true; } @@ -188,8 +133,6 @@ __visible bool ex_handler_ext(const struct exception_table_entry *fixup, unsigned long error_code, unsigned long fault_addr) { - if (bogus_uaccess(regs, trapnr, fault_addr)) - return false; /* Special hack for uaccess_err */ current->thread.uaccess_err = 1; regs->ip = ex_fixup_addr(fixup); diff --git a/arch/xtensa/include/asm/asm-uaccess.h b/arch/xtensa/include/asm/asm-uaccess.h index dfdf9fae1f84..7f6cf4151843 100644 --- a/arch/xtensa/include/asm/asm-uaccess.h +++ b/arch/xtensa/include/asm/asm-uaccess.h @@ -32,8 +32,6 @@ #define KERNEL_DS 0 #define USER_DS 1 -#define get_ds (KERNEL_DS) - /* * get_fs reads current->thread.current_ds into a register. * On Entry: diff --git a/arch/xtensa/include/asm/uaccess.h b/arch/xtensa/include/asm/uaccess.h index 4b2480304bc3..6792928ba84a 100644 --- a/arch/xtensa/include/asm/uaccess.h +++ b/arch/xtensa/include/asm/uaccess.h @@ -32,7 +32,6 @@ #define KERNEL_DS ((mm_segment_t) { 0 }) #define USER_DS ((mm_segment_t) { 1 }) -#define get_ds() (KERNEL_DS) #define get_fs() (current->thread.current_ds) #define set_fs(val) (current->thread.current_ds = (val)) diff --git a/drivers/crypto/ccree/cc_pm.h b/drivers/crypto/ccree/cc_pm.h index f62624357020..907a6db4d6c0 100644 --- a/drivers/crypto/ccree/cc_pm.h +++ b/drivers/crypto/ccree/cc_pm.h @@ -30,7 +30,7 @@ static inline int cc_pm_init(struct cc_drvdata *drvdata) return 0; } -static void cc_pm_go(struct cc_drvdata *drvdata) {} +static inline void cc_pm_go(struct cc_drvdata *drvdata) {} static inline void cc_pm_fini(struct cc_drvdata *drvdata) {} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h index aadd0fa42e43..3aa42c64484a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h @@ -405,6 +405,7 @@ struct amdgpu_crtc { struct amdgpu_flip_work *pflip_works; enum amdgpu_flip_status pflip_status; int deferred_flip_completion; + u64 last_flip_vblank; /* pll sharing */ struct amdgpu_atom_ss ss; bool ss_enabled; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index 5296b8f3e0ab..636d14a60952 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -303,12 +303,11 @@ static void dm_pflip_high_irq(void *interrupt_params) return; } + /* Update to correct count(s) if racing with vblank irq */ + amdgpu_crtc->last_flip_vblank = drm_crtc_accurate_vblank_count(&amdgpu_crtc->base); /* wake up userspace */ if (amdgpu_crtc->event) { - /* Update to correct count(s) if racing with vblank irq */ - drm_crtc_accurate_vblank_count(&amdgpu_crtc->base); - drm_crtc_send_vblank_event(&amdgpu_crtc->base, amdgpu_crtc->event); /* page flip completed. clean up */ @@ -4828,6 +4827,8 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, to_dm_crtc_state(drm_atomic_get_old_crtc_state(state, pcrtc)); int planes_count = 0; unsigned long flags; + u64 last_flip_vblank; + bool vrr_active = acrtc_state->freesync_config.state == VRR_STATE_ACTIVE_VARIABLE; /* update planes when needed */ for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, i) { @@ -4859,6 +4860,16 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, /* In commit tail framework this cannot happen */ WARN_ON(1); } + + /* For variable refresh rate mode only: + * Get vblank of last completed flip to avoid > 1 vrr flips per + * video frame by use of throttling, but allow flip programming + * anywhere in the possibly large variable vrr vblank interval + * for fine-grained flip timing control and more opportunity to + * avoid stutter on late submission of amdgpu_dm_do_flip() calls. + */ + last_flip_vblank = acrtc_attach->last_flip_vblank; + spin_unlock_irqrestore(&crtc->dev->event_lock, flags); if (!pflip_needed || plane->type == DRM_PLANE_TYPE_OVERLAY) { @@ -4882,10 +4893,18 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, if (plane->type == DRM_PLANE_TYPE_PRIMARY) drm_crtc_vblank_get(crtc); + /* Use old throttling in non-vrr fixed refresh rate mode + * to keep flip scheduling based on target vblank counts + * working in a backwards compatible way, e.g., clients + * using GLX_OML_sync_control extension. + */ + if (!vrr_active) + last_flip_vblank = drm_crtc_vblank_count(crtc); + amdgpu_dm_do_flip( crtc, fb, - (uint32_t)drm_crtc_vblank_count(crtc) + *wait_for_vblank, + (uint32_t) last_flip_vblank + *wait_for_vblank, dc_state); } diff --git a/drivers/gpu/drm/bochs/bochs_drv.c b/drivers/gpu/drm/bochs/bochs_drv.c index f3dd66ae990a..aa35007262cd 100644 --- a/drivers/gpu/drm/bochs/bochs_drv.c +++ b/drivers/gpu/drm/bochs/bochs_drv.c @@ -154,6 +154,10 @@ static int bochs_pci_probe(struct pci_dev *pdev, if (IS_ERR(dev)) return PTR_ERR(dev); + ret = pci_enable_device(pdev); + if (ret) + goto err_free_dev; + dev->pdev = pdev; pci_set_drvdata(pdev, dev); diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c index 54e2ae614dcc..f4290f6b0c38 100644 --- a/drivers/gpu/drm/drm_atomic_helper.c +++ b/drivers/gpu/drm/drm_atomic_helper.c @@ -1602,6 +1602,15 @@ int drm_atomic_helper_async_check(struct drm_device *dev, old_plane_state->crtc != new_plane_state->crtc) return -EINVAL; + /* + * FIXME: Since prepare_fb and cleanup_fb are always called on + * the new_plane_state for async updates we need to block framebuffer + * changes. This prevents use of a fb that's been cleaned up and + * double cleanups from occuring. + */ + if (old_plane_state->fb != new_plane_state->fb) + return -EINVAL; + funcs = plane->helper_private; if (!funcs->atomic_async_update) return -EINVAL; diff --git a/drivers/i3c/master/dw-i3c-master.c b/drivers/i3c/master/dw-i3c-master.c index bb03079fbade..59279224e07f 100644 --- a/drivers/i3c/master/dw-i3c-master.c +++ b/drivers/i3c/master/dw-i3c-master.c @@ -602,6 +602,7 @@ static int dw_i3c_master_bus_init(struct i3c_master_controller *m) ret = dw_i2c_clk_cfg(master); if (ret) return ret; + /* fall through */ case I3C_BUS_MODE_PURE: ret = dw_i3c_clk_cfg(master); if (ret) diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index dc9f14811e0f..58dc70bffd5b 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -144,7 +144,7 @@ dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event) for (tmp = dev; tmp; tmp = tmp->bus->self) level++; - size = sizeof(*info) + level * sizeof(struct acpi_dmar_pci_path); + size = sizeof(*info) + level * sizeof(info->path[0]); if (size <= sizeof(dmar_pci_notify_info_buf)) { info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf; } else { diff --git a/drivers/mtd/chips/cfi_cmdset_0001.c b/drivers/mtd/chips/cfi_cmdset_0001.c index 6e8e7b1bb34b..79a53cb8507b 100644 --- a/drivers/mtd/chips/cfi_cmdset_0001.c +++ b/drivers/mtd/chips/cfi_cmdset_0001.c @@ -756,7 +756,8 @@ static int cfi_intelext_partition_fixup(struct mtd_info *mtd, } numvirtchips = cfi->numchips * numparts; - newcfi = kmalloc(sizeof(struct cfi_private) + numvirtchips * sizeof(struct flchip), GFP_KERNEL); + newcfi = kmalloc(struct_size(newcfi, chips, numvirtchips), + GFP_KERNEL); if (!newcfi) return -ENOMEM; shared = kmalloc_array(cfi->numchips, diff --git a/drivers/mtd/chips/gen_probe.c b/drivers/mtd/chips/gen_probe.c index 837b04ab96a9..839ed40625d6 100644 --- a/drivers/mtd/chips/gen_probe.c +++ b/drivers/mtd/chips/gen_probe.c @@ -135,7 +135,7 @@ static struct cfi_private *genprobe_ident_chips(struct map_info *map, struct chi * our caller, and copy the appropriate data into them. */ - retcfi = kmalloc(sizeof(struct cfi_private) + cfi.numchips * sizeof(struct flchip), GFP_KERNEL); + retcfi = kmalloc(struct_size(retcfi, chips, cfi.numchips), GFP_KERNEL); if (!retcfi) { kfree(cfi.cfiq); diff --git a/drivers/mtd/devices/docg3.c b/drivers/mtd/devices/docg3.c index 4c94fc096696..7754803e3463 100644 --- a/drivers/mtd/devices/docg3.c +++ b/drivers/mtd/devices/docg3.c @@ -1767,8 +1767,8 @@ static int __init doc_set_driver_info(int chip_id, struct mtd_info *mtd) switch (chip_id) { case DOC_CHIPID_G3: - mtd->name = kasprintf(GFP_KERNEL, "docg3.%d", - docg3->device_id); + mtd->name = devm_kasprintf(docg3->dev, GFP_KERNEL, "docg3.%d", + docg3->device_id); if (!mtd->name) return -ENOMEM; docg3->max_block = 2047; @@ -1872,7 +1872,7 @@ nomem3: nomem2: kfree(docg3); nomem1: - return ERR_PTR(ret); + return ret ? ERR_PTR(ret) : NULL; } /** @@ -1886,7 +1886,6 @@ static void doc_release_device(struct mtd_info *mtd) mtd_device_unregister(mtd); kfree(docg3->bbt); kfree(docg3); - kfree(mtd->name); kfree(mtd); } diff --git a/drivers/mtd/devices/m25p80.c b/drivers/mtd/devices/m25p80.c index c4a1d04b8c80..651bab6d4e31 100644 --- a/drivers/mtd/devices/m25p80.c +++ b/drivers/mtd/devices/m25p80.c @@ -195,7 +195,14 @@ static int m25p_probe(struct spi_mem *spimem) spi_mem_set_drvdata(spimem, flash); flash->spimem = spimem; - if (spi->mode & SPI_RX_QUAD) { + if (spi->mode & SPI_RX_OCTAL) { + hwcaps.mask |= SNOR_HWCAPS_READ_1_1_8; + + if (spi->mode & SPI_TX_OCTAL) + hwcaps.mask |= (SNOR_HWCAPS_READ_1_8_8 | + SNOR_HWCAPS_PP_1_1_8 | + SNOR_HWCAPS_PP_1_8_8); + } else if (spi->mode & SPI_RX_QUAD) { hwcaps.mask |= SNOR_HWCAPS_READ_1_1_4; if (spi->mode & SPI_TX_QUAD) diff --git a/drivers/mtd/devices/mtdram.c b/drivers/mtd/devices/mtdram.c index 46238796145f..1c97fabc4bf9 100644 --- a/drivers/mtd/devices/mtdram.c +++ b/drivers/mtd/devices/mtdram.c @@ -24,14 +24,12 @@ static unsigned long writebuf_size = 64; #define MTDRAM_TOTAL_SIZE (total_size * 1024) #define MTDRAM_ERASE_SIZE (erase_size * 1024) -#ifdef MODULE module_param(total_size, ulong, 0); MODULE_PARM_DESC(total_size, "Total device size in KiB"); module_param(erase_size, ulong, 0); MODULE_PARM_DESC(erase_size, "Device erase block size in KiB"); module_param(writebuf_size, ulong, 0); MODULE_PARM_DESC(writebuf_size, "Device write buf size in Bytes (Default: 64)"); -#endif // We could store these in the mtd structure, but we only support 1 device.. static struct mtd_info *mtd_info; diff --git a/drivers/mtd/lpddr/qinfo_probe.c b/drivers/mtd/lpddr/qinfo_probe.c index 69f2112340b1..175bdc3b72f4 100644 --- a/drivers/mtd/lpddr/qinfo_probe.c +++ b/drivers/mtd/lpddr/qinfo_probe.c @@ -181,8 +181,8 @@ static struct lpddr_private *lpddr_probe_chip(struct map_info *map) lpddr.numchips = 1; numvirtchips = lpddr.numchips * lpddr.qinfo->HWPartsNum; - retlpddr = kzalloc(sizeof(struct lpddr_private) + - numvirtchips * sizeof(struct flchip), GFP_KERNEL); + retlpddr = kzalloc(struct_size(retlpddr, chips, numvirtchips), + GFP_KERNEL); if (!retlpddr) return NULL; diff --git a/drivers/mtd/mtdcore.c b/drivers/mtd/mtdcore.c index 3ef01baef9b6..76b4264936ff 100644 --- a/drivers/mtd/mtdcore.c +++ b/drivers/mtd/mtdcore.c @@ -155,7 +155,6 @@ static ssize_t mtd_flags_show(struct device *dev, struct mtd_info *mtd = dev_get_drvdata(dev); return snprintf(buf, PAGE_SIZE, "0x%lx\n", (unsigned long)mtd->flags); - } static DEVICE_ATTR(flags, S_IRUGO, mtd_flags_show, NULL); @@ -166,7 +165,6 @@ static ssize_t mtd_size_show(struct device *dev, return snprintf(buf, PAGE_SIZE, "%llu\n", (unsigned long long)mtd->size); - } static DEVICE_ATTR(size, S_IRUGO, mtd_size_show, NULL); @@ -176,7 +174,6 @@ static ssize_t mtd_erasesize_show(struct device *dev, struct mtd_info *mtd = dev_get_drvdata(dev); return snprintf(buf, PAGE_SIZE, "%lu\n", (unsigned long)mtd->erasesize); - } static DEVICE_ATTR(erasesize, S_IRUGO, mtd_erasesize_show, NULL); @@ -186,7 +183,6 @@ static ssize_t mtd_writesize_show(struct device *dev, struct mtd_info *mtd = dev_get_drvdata(dev); return snprintf(buf, PAGE_SIZE, "%lu\n", (unsigned long)mtd->writesize); - } static DEVICE_ATTR(writesize, S_IRUGO, mtd_writesize_show, NULL); @@ -197,7 +193,6 @@ static ssize_t mtd_subpagesize_show(struct device *dev, unsigned int subpagesize = mtd->writesize >> mtd->subpage_sft; return snprintf(buf, PAGE_SIZE, "%u\n", subpagesize); - } static DEVICE_ATTR(subpagesize, S_IRUGO, mtd_subpagesize_show, NULL); @@ -207,7 +202,6 @@ static ssize_t mtd_oobsize_show(struct device *dev, struct mtd_info *mtd = dev_get_drvdata(dev); return snprintf(buf, PAGE_SIZE, "%lu\n", (unsigned long)mtd->oobsize); - } static DEVICE_ATTR(oobsize, S_IRUGO, mtd_oobsize_show, NULL); @@ -226,7 +220,6 @@ static ssize_t mtd_numeraseregions_show(struct device *dev, struct mtd_info *mtd = dev_get_drvdata(dev); return snprintf(buf, PAGE_SIZE, "%u\n", mtd->numeraseregions); - } static DEVICE_ATTR(numeraseregions, S_IRUGO, mtd_numeraseregions_show, NULL); @@ -237,7 +230,6 @@ static ssize_t mtd_name_show(struct device *dev, struct mtd_info *mtd = dev_get_drvdata(dev); return snprintf(buf, PAGE_SIZE, "%s\n", mtd->name); - } static DEVICE_ATTR(name, S_IRUGO, mtd_name_show, NULL); @@ -560,6 +552,14 @@ int add_mtd_device(struct mtd_info *mtd) BUG_ON(mtd->writesize == 0); + /* + * MTD drivers should implement ->_{write,read}() or + * ->_{write,read}_oob(), but not both. + */ + if (WARN_ON((mtd->_write && mtd->_write_oob) || + (mtd->_read && mtd->_read_oob))) + return -EINVAL; + if (WARN_ON((!mtd->erasesize || !mtd->_erase) && !(mtd->flags & MTD_NO_ERASE))) return -EINVAL; @@ -1090,67 +1090,32 @@ EXPORT_SYMBOL_GPL(mtd_get_unmapped_area); int mtd_read(struct mtd_info *mtd, loff_t from, size_t len, size_t *retlen, u_char *buf) { - int ret_code; - *retlen = 0; - if (from < 0 || from >= mtd->size || len > mtd->size - from) - return -EINVAL; - if (!len) - return 0; + struct mtd_oob_ops ops = { + .len = len, + .datbuf = buf, + }; + int ret; - ledtrig_mtd_activity(); - /* - * In the absence of an error, drivers return a non-negative integer - * representing the maximum number of bitflips that were corrected on - * any one ecc region (if applicable; zero otherwise). - */ - if (mtd->_read) { - ret_code = mtd->_read(mtd, from, len, retlen, buf); - } else if (mtd->_read_oob) { - struct mtd_oob_ops ops = { - .len = len, - .datbuf = buf, - }; - - ret_code = mtd->_read_oob(mtd, from, &ops); - *retlen = ops.retlen; - } else { - return -ENOTSUPP; - } + ret = mtd_read_oob(mtd, from, &ops); + *retlen = ops.retlen; - if (unlikely(ret_code < 0)) - return ret_code; - if (mtd->ecc_strength == 0) - return 0; /* device lacks ecc */ - return ret_code >= mtd->bitflip_threshold ? -EUCLEAN : 0; + return ret; } EXPORT_SYMBOL_GPL(mtd_read); int mtd_write(struct mtd_info *mtd, loff_t to, size_t len, size_t *retlen, const u_char *buf) { - *retlen = 0; - if (to < 0 || to >= mtd->size || len > mtd->size - to) - return -EINVAL; - if ((!mtd->_write && !mtd->_write_oob) || - !(mtd->flags & MTD_WRITEABLE)) - return -EROFS; - if (!len) - return 0; - ledtrig_mtd_activity(); + struct mtd_oob_ops ops = { + .len = len, + .datbuf = (u8 *)buf, + }; + int ret; - if (!mtd->_write) { - struct mtd_oob_ops ops = { - .len = len, - .datbuf = (u8 *)buf, - }; - int ret; + ret = mtd_write_oob(mtd, to, &ops); + *retlen = ops.retlen; - ret = mtd->_write_oob(mtd, to, &ops); - *retlen = ops.retlen; - return ret; - } - - return mtd->_write(mtd, to, len, retlen, buf); + return ret; } EXPORT_SYMBOL_GPL(mtd_write); diff --git a/drivers/mtd/nand/raw/Kconfig b/drivers/mtd/nand/raw/Kconfig index 1a55d3e3d4c5..e604625e2dfa 100644 --- a/drivers/mtd/nand/raw/Kconfig +++ b/drivers/mtd/nand/raw/Kconfig @@ -541,4 +541,21 @@ config MTD_NAND_TEGRA is supported. Extra OOB bytes when using HW ECC are currently not supported. +config MTD_NAND_STM32_FMC2 + tristate "Support for NAND controller on STM32MP SoCs" + depends on MACH_STM32MP157 || COMPILE_TEST + help + Enables support for NAND Flash chips on SoCs containing the FMC2 + NAND controller. This controller is found on STM32MP SoCs. + The controller supports a maximum 8k page size and supports + a maximum 8-bit correction error per sector of 512 bytes. + +config MTD_NAND_MESON + tristate "Support for NAND controller on Amlogic's Meson SoCs" + depends on ARCH_MESON || COMPILE_TEST + select MFD_SYSCON + help + Enables support for NAND controller on Amlogic's Meson SoCs. + This controller is found on Meson SoCs. + endif # MTD_NAND diff --git a/drivers/mtd/nand/raw/Makefile b/drivers/mtd/nand/raw/Makefile index 57159b349054..5a5a72f0793e 100644 --- a/drivers/mtd/nand/raw/Makefile +++ b/drivers/mtd/nand/raw/Makefile @@ -56,6 +56,8 @@ obj-$(CONFIG_MTD_NAND_BRCMNAND) += brcmnand/ obj-$(CONFIG_MTD_NAND_QCOM) += qcom_nandc.o obj-$(CONFIG_MTD_NAND_MTK) += mtk_ecc.o mtk_nand.o obj-$(CONFIG_MTD_NAND_TEGRA) += tegra_nand.o +obj-$(CONFIG_MTD_NAND_STM32_FMC2) += stm32_fmc2_nand.o +obj-$(CONFIG_MTD_NAND_MESON) += meson_nand.o nand-objs := nand_base.o nand_legacy.o nand_bbt.o nand_timings.o nand_ids.o nand-objs += nand_onfi.o diff --git a/drivers/mtd/nand/raw/atmel/pmecc.c b/drivers/mtd/nand/raw/atmel/pmecc.c index 555a74e15269..9d3997840889 100644 --- a/drivers/mtd/nand/raw/atmel/pmecc.c +++ b/drivers/mtd/nand/raw/atmel/pmecc.c @@ -876,23 +876,32 @@ static struct atmel_pmecc *atmel_pmecc_get_by_node(struct device *userdev, { struct platform_device *pdev; struct atmel_pmecc *pmecc, **ptr; + int ret; pdev = of_find_device_by_node(np); - if (!pdev || !platform_get_drvdata(pdev)) + if (!pdev) return ERR_PTR(-EPROBE_DEFER); + pmecc = platform_get_drvdata(pdev); + if (!pmecc) { + ret = -EPROBE_DEFER; + goto err_put_device; + } ptr = devres_alloc(devm_atmel_pmecc_put, sizeof(*ptr), GFP_KERNEL); - if (!ptr) - return ERR_PTR(-ENOMEM); - - get_device(&pdev->dev); - pmecc = platform_get_drvdata(pdev); + if (!ptr) { + ret = -ENOMEM; + goto err_put_device; + } *ptr = pmecc; devres_add(userdev, ptr); return pmecc; + +err_put_device: + put_device(&pdev->dev); + return ERR_PTR(ret); } static const int atmel_pmecc_strengths[] = { 2, 4, 8, 12, 24, 32 }; diff --git a/drivers/mtd/nand/raw/denali.c b/drivers/mtd/nand/raw/denali.c index 6e8edc9375dd..24aeafc67cd4 100644 --- a/drivers/mtd/nand/raw/denali.c +++ b/drivers/mtd/nand/raw/denali.c @@ -37,9 +37,6 @@ #define DENALI_MAP11_ADDR ((DENALI_MAP11) | 1) /* address cycle */ #define DENALI_MAP11_DATA ((DENALI_MAP11) | 2) /* data cycle */ -/* MAP10 commands */ -#define DENALI_ERASE 0x01 - #define DENALI_BANK(denali) ((denali)->active_bank << 24) #define DENALI_INVALID_BANK -1 @@ -476,7 +473,7 @@ static void denali_setup_dma32(struct denali_nand_info *denali, } static int denali_pio_read(struct denali_nand_info *denali, void *buf, - size_t size, int page, int raw) + size_t size, int page) { u32 addr = DENALI_MAP01 | DENALI_BANK(denali) | page; uint32_t *buf32 = (uint32_t *)buf; @@ -504,7 +501,7 @@ static int denali_pio_read(struct denali_nand_info *denali, void *buf, } static int denali_pio_write(struct denali_nand_info *denali, - const void *buf, size_t size, int page, int raw) + const void *buf, size_t size, int page) { u32 addr = DENALI_MAP01 | DENALI_BANK(denali) | page; const uint32_t *buf32 = (uint32_t *)buf; @@ -525,16 +522,16 @@ static int denali_pio_write(struct denali_nand_info *denali, } static int denali_pio_xfer(struct denali_nand_info *denali, void *buf, - size_t size, int page, int raw, int write) + size_t size, int page, int write) { if (write) - return denali_pio_write(denali, buf, size, page, raw); + return denali_pio_write(denali, buf, size, page); else - return denali_pio_read(denali, buf, size, page, raw); + return denali_pio_read(denali, buf, size, page); } static int denali_dma_xfer(struct denali_nand_info *denali, void *buf, - size_t size, int page, int raw, int write) + size_t size, int page, int write) { dma_addr_t dma_addr; uint32_t irq_mask, irq_status, ecc_err_mask; @@ -544,7 +541,7 @@ static int denali_dma_xfer(struct denali_nand_info *denali, void *buf, dma_addr = dma_map_single(denali->dev, buf, size, dir); if (dma_mapping_error(denali->dev, dma_addr)) { dev_dbg(denali->dev, "Failed to DMA-map buffer. Trying PIO.\n"); - return denali_pio_xfer(denali, buf, size, page, raw, write); + return denali_pio_xfer(denali, buf, size, page, write); } if (write) { @@ -598,9 +595,9 @@ static int denali_data_xfer(struct denali_nand_info *denali, void *buf, denali->reg + TRANSFER_SPARE_REG); if (denali->dma_avail) - return denali_dma_xfer(denali, buf, size, page, raw, write); + return denali_dma_xfer(denali, buf, size, page, write); else - return denali_pio_xfer(denali, buf, size, page, raw, write); + return denali_pio_xfer(denali, buf, size, page, write); } static void denali_oob_xfer(struct mtd_info *mtd, struct nand_chip *chip, @@ -754,9 +751,6 @@ static int denali_read_oob(struct nand_chip *chip, int page) static int denali_write_oob(struct nand_chip *chip, int page) { struct mtd_info *mtd = nand_to_mtd(chip); - struct denali_nand_info *denali = mtd_to_denali(mtd); - - denali_reset_irq(denali); denali_oob_xfer(mtd, chip, page, 1); @@ -903,23 +897,6 @@ static int denali_waitfunc(struct nand_chip *chip) return irq_status & INTR__INT_ACT ? 0 : NAND_STATUS_FAIL; } -static int denali_erase(struct nand_chip *chip, int page) -{ - struct denali_nand_info *denali = mtd_to_denali(nand_to_mtd(chip)); - uint32_t irq_status; - - denali_reset_irq(denali); - - denali->host_write(denali, DENALI_MAP10 | DENALI_BANK(denali) | page, - DENALI_ERASE); - - /* wait for erase to complete or failure to occur */ - irq_status = denali_wait_for_irq(denali, - INTR__ERASE_COMP | INTR__ERASE_FAIL); - - return irq_status & INTR__ERASE_COMP ? 0 : -EIO; -} - static int denali_setup_data_interface(struct nand_chip *chip, int chipnr, const struct nand_data_interface *conf) { @@ -1244,7 +1221,6 @@ static int denali_attach_chip(struct nand_chip *chip) chip->ecc.write_page_raw = denali_write_page_raw; chip->ecc.read_oob = denali_read_oob; chip->ecc.write_oob = denali_write_oob; - chip->legacy.erase = denali_erase; ret = denali_multidev_fixup(denali); if (ret) diff --git a/drivers/mtd/nand/raw/denali.h b/drivers/mtd/nand/raw/denali.h index 25c00601b8b3..c8c2620fc736 100644 --- a/drivers/mtd/nand/raw/denali.h +++ b/drivers/mtd/nand/raw/denali.h @@ -304,7 +304,6 @@ struct denali_nand_info { u32 irq_status; /* interrupts that have happened */ int irq; void *buf; /* for syndrome layout conversion */ - dma_addr_t dma_addr; int dma_avail; /* can support DMA? */ int devs_per_cs; /* devices connected in parallel */ int oob_skip_bytes; /* number of bytes reserved for BBM */ diff --git a/drivers/mtd/nand/raw/denali_dt.c b/drivers/mtd/nand/raw/denali_dt.c index 7c6a8a426606..0b5ae2418815 100644 --- a/drivers/mtd/nand/raw/denali_dt.c +++ b/drivers/mtd/nand/raw/denali_dt.c @@ -109,25 +109,17 @@ static int denali_dt_probe(struct platform_device *pdev) if (IS_ERR(denali->host)) return PTR_ERR(denali->host); - /* - * A single anonymous clock is supported for the backward compatibility. - * New platforms should support all the named clocks. - */ dt->clk = devm_clk_get(dev, "nand"); if (IS_ERR(dt->clk)) - dt->clk = devm_clk_get(dev, NULL); - if (IS_ERR(dt->clk)) { - dev_err(dev, "no clk available\n"); return PTR_ERR(dt->clk); - } dt->clk_x = devm_clk_get(dev, "nand_x"); if (IS_ERR(dt->clk_x)) - dt->clk_x = NULL; + return PTR_ERR(dt->clk_x); dt->clk_ecc = devm_clk_get(dev, "ecc"); if (IS_ERR(dt->clk_ecc)) - dt->clk_ecc = NULL; + return PTR_ERR(dt->clk_ecc); ret = clk_prepare_enable(dt->clk); if (ret) @@ -141,19 +133,8 @@ static int denali_dt_probe(struct platform_device *pdev) if (ret) goto out_disable_clk_x; - if (dt->clk_x) { - denali->clk_rate = clk_get_rate(dt->clk); - denali->clk_x_rate = clk_get_rate(dt->clk_x); - } else { - /* - * Hardcode the clock rates for the backward compatibility. - * This works for both SOCFPGA and UniPhier. - */ - dev_notice(dev, - "necessary clock is missing. default clock rates are used.\n"); - denali->clk_rate = 50000000; - denali->clk_x_rate = 200000000; - } + denali->clk_rate = clk_get_rate(dt->clk); + denali->clk_x_rate = clk_get_rate(dt->clk_x); ret = denali_init(denali); if (ret) diff --git a/drivers/mtd/nand/raw/fsmc_nand.c b/drivers/mtd/nand/raw/fsmc_nand.c index c9149a37f8f0..6c7ca41354be 100644 --- a/drivers/mtd/nand/raw/fsmc_nand.c +++ b/drivers/mtd/nand/raw/fsmc_nand.c @@ -965,6 +965,19 @@ static const struct nand_controller_ops fsmc_nand_controller_ops = { .setup_data_interface = fsmc_setup_data_interface, }; +/** + * fsmc_nand_disable() - Disables the NAND bank + * @host: The instance to disable + */ +static void fsmc_nand_disable(struct fsmc_nand_data *host) +{ + u32 val; + + val = readl(host->regs_va + FSMC_PC); + val &= ~FSMC_ENABLE; + writel(val, host->regs_va + FSMC_PC); +} + /* * fsmc_nand_probe - Probe function * @pdev: platform device structure @@ -1120,6 +1133,7 @@ release_dma_read_chan: if (host->mode == USE_DMA_ACCESS) dma_release_channel(host->read_dma_chan); disable_clk: + fsmc_nand_disable(host); clk_disable_unprepare(host->clk); return ret; @@ -1134,6 +1148,7 @@ static int fsmc_nand_remove(struct platform_device *pdev) if (host) { nand_release(&host->nand); + fsmc_nand_disable(host); if (host->mode == USE_DMA_ACCESS) { dma_release_channel(host->write_dma_chan); @@ -1164,6 +1179,7 @@ static int fsmc_nand_resume(struct device *dev) clk_prepare_enable(host->clk); if (host->dev_timings) fsmc_nand_setup(host, host->dev_timings); + nand_reset(&host->nand, 0); } return 0; diff --git a/drivers/mtd/nand/raw/jz4780_bch.c b/drivers/mtd/nand/raw/jz4780_bch.c index 7201827809e9..c5f74ed85862 100644 --- a/drivers/mtd/nand/raw/jz4780_bch.c +++ b/drivers/mtd/nand/raw/jz4780_bch.c @@ -281,12 +281,15 @@ static struct jz4780_bch *jz4780_bch_get(struct device_node *np) struct jz4780_bch *bch; pdev = of_find_device_by_node(np); - if (!pdev || !platform_get_drvdata(pdev)) + if (!pdev) return ERR_PTR(-EPROBE_DEFER); - get_device(&pdev->dev); - bch = platform_get_drvdata(pdev); + if (!bch) { + put_device(&pdev->dev); + return ERR_PTR(-EPROBE_DEFER); + } + clk_prepare_enable(bch->clk); return bch; diff --git a/drivers/mtd/nand/raw/marvell_nand.c b/drivers/mtd/nand/raw/marvell_nand.c index 84283c6bb0ff..f38e5c1b87e4 100644 --- a/drivers/mtd/nand/raw/marvell_nand.c +++ b/drivers/mtd/nand/raw/marvell_nand.c @@ -2550,9 +2550,8 @@ static int marvell_nand_chip_init(struct device *dev, struct marvell_nfc *nfc, } /* Alloc the nand chip structure */ - marvell_nand = devm_kzalloc(dev, sizeof(*marvell_nand) + - (nsels * - sizeof(struct marvell_nand_chip_sel)), + marvell_nand = devm_kzalloc(dev, + struct_size(marvell_nand, sels, nsels), GFP_KERNEL); if (!marvell_nand) { dev_err(dev, "could not allocate chip structure\n"); diff --git a/drivers/mtd/nand/raw/meson_nand.c b/drivers/mtd/nand/raw/meson_nand.c new file mode 100644 index 000000000000..3e8aa71407b5 --- /dev/null +++ b/drivers/mtd/nand/raw/meson_nand.c @@ -0,0 +1,1464 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Amlogic Meson Nand Flash Controller Driver + * + * Copyright (c) 2018 Amlogic, inc. + * Author: Liang Yang <liang.yang@amlogic.com> + */ + +#include <linux/platform_device.h> +#include <linux/dma-mapping.h> +#include <linux/interrupt.h> +#include <linux/clk.h> +#include <linux/mtd/rawnand.h> +#include <linux/mtd/mtd.h> +#include <linux/mfd/syscon.h> +#include <linux/regmap.h> +#include <linux/slab.h> +#include <linux/module.h> +#include <linux/iopoll.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/sched/task_stack.h> + +#define NFC_REG_CMD 0x00 +#define NFC_CMD_IDLE (0xc << 14) +#define NFC_CMD_CLE (0x5 << 14) +#define NFC_CMD_ALE (0x6 << 14) +#define NFC_CMD_ADL ((0 << 16) | (3 << 20)) +#define NFC_CMD_ADH ((1 << 16) | (3 << 20)) +#define NFC_CMD_AIL ((2 << 16) | (3 << 20)) +#define NFC_CMD_AIH ((3 << 16) | (3 << 20)) +#define NFC_CMD_SEED ((8 << 16) | (3 << 20)) +#define NFC_CMD_M2N ((0 << 17) | (2 << 20)) +#define NFC_CMD_N2M ((1 << 17) | (2 << 20)) +#define NFC_CMD_RB BIT(20) +#define NFC_CMD_SCRAMBLER_ENABLE BIT(19) +#define NFC_CMD_SCRAMBLER_DISABLE 0 +#define NFC_CMD_SHORTMODE_DISABLE 0 +#define NFC_CMD_RB_INT BIT(14) + +#define NFC_CMD_GET_SIZE(x) (((x) >> 22) & GENMASK(4, 0)) + +#define NFC_REG_CFG 0x04 +#define NFC_REG_DADR 0x08 +#define NFC_REG_IADR 0x0c +#define NFC_REG_BUF 0x10 +#define NFC_REG_INFO 0x14 +#define NFC_REG_DC 0x18 +#define NFC_REG_ADR 0x1c +#define NFC_REG_DL 0x20 +#define NFC_REG_DH 0x24 +#define NFC_REG_CADR 0x28 +#define NFC_REG_SADR 0x2c +#define NFC_REG_PINS 0x30 +#define NFC_REG_VER 0x38 + +#define NFC_RB_IRQ_EN BIT(21) + +#define CMDRWGEN(cmd_dir, ran, bch, short_mode, page_size, pages) \ + ( \ + (cmd_dir) | \ + ((ran) << 19) | \ + ((bch) << 14) | \ + ((short_mode) << 13) | \ + (((page_size) & 0x7f) << 6) | \ + ((pages) & 0x3f) \ + ) + +#define GENCMDDADDRL(adl, addr) ((adl) | ((addr) & 0xffff)) +#define GENCMDDADDRH(adh, addr) ((adh) | (((addr) >> 16) & 0xffff)) +#define GENCMDIADDRL(ail, addr) ((ail) | ((addr) & 0xffff)) +#define GENCMDIADDRH(aih, addr) ((aih) | (((addr) >> 16) & 0xffff)) + +#define DMA_DIR(dir) ((dir) ? NFC_CMD_N2M : NFC_CMD_M2N) + +#define ECC_CHECK_RETURN_FF (-1) + +#define NAND_CE0 (0xe << 10) +#define NAND_CE1 (0xd << 10) + +#define DMA_BUSY_TIMEOUT 0x100000 +#define CMD_FIFO_EMPTY_TIMEOUT 1000 + +#define MAX_CE_NUM 2 + +/* eMMC clock register, misc control */ +#define CLK_SELECT_NAND BIT(31) + +#define NFC_CLK_CYCLE 6 + +/* nand flash controller delay 3 ns */ +#define NFC_DEFAULT_DELAY 3000 + +#define ROW_ADDER(page, index) (((page) >> (8 * (index))) & 0xff) +#define MAX_CYCLE_ADDRS 5 +#define DIRREAD 1 +#define DIRWRITE 0 + +#define ECC_PARITY_BCH8_512B 14 +#define ECC_COMPLETE BIT(31) +#define ECC_ERR_CNT(x) (((x) >> 24) & GENMASK(5, 0)) +#define ECC_ZERO_CNT(x) (((x) >> 16) & GENMASK(5, 0)) +#define ECC_UNCORRECTABLE 0x3f + +#define PER_INFO_BYTE 8 + +struct meson_nfc_nand_chip { + struct list_head node; + struct nand_chip nand; + unsigned long clk_rate; + unsigned long level1_divider; + u32 bus_timing; + u32 twb; + u32 tadl; + u32 tbers_max; + + u32 bch_mode; + u8 *data_buf; + __le64 *info_buf; + u32 nsels; + u8 sels[0]; +}; + +struct meson_nand_ecc { + u32 bch; + u32 strength; +}; + +struct meson_nfc_data { + const struct nand_ecc_caps *ecc_caps; +}; + +struct meson_nfc_param { + u32 chip_select; + u32 rb_select; +}; + +struct nand_rw_cmd { + u32 cmd0; + u32 addrs[MAX_CYCLE_ADDRS]; + u32 cmd1; +}; + +struct nand_timing { + u32 twb; + u32 tadl; + u32 tbers_max; +}; + +struct meson_nfc { + struct nand_controller controller; + struct clk *core_clk; + struct clk *device_clk; + struct clk *phase_tx; + struct clk *phase_rx; + + unsigned long clk_rate; + u32 bus_timing; + + struct device *dev; + void __iomem *reg_base; + struct regmap *reg_clk; + struct completion completion; + struct list_head chips; + const struct meson_nfc_data *data; + struct meson_nfc_param param; + struct nand_timing timing; + union { + int cmd[32]; + struct nand_rw_cmd rw; + } cmdfifo; + + dma_addr_t daddr; + dma_addr_t iaddr; + + unsigned long assigned_cs; +}; + +enum { + NFC_ECC_BCH8_1K = 2, + NFC_ECC_BCH24_1K, + NFC_ECC_BCH30_1K, + NFC_ECC_BCH40_1K, + NFC_ECC_BCH50_1K, + NFC_ECC_BCH60_1K, +}; + +#define MESON_ECC_DATA(b, s) { .bch = (b), .strength = (s)} + +static struct meson_nand_ecc meson_ecc[] = { + MESON_ECC_DATA(NFC_ECC_BCH8_1K, 8), + MESON_ECC_DATA(NFC_ECC_BCH24_1K, 24), + MESON_ECC_DATA(NFC_ECC_BCH30_1K, 30), + MESON_ECC_DATA(NFC_ECC_BCH40_1K, 40), + MESON_ECC_DATA(NFC_ECC_BCH50_1K, 50), + MESON_ECC_DATA(NFC_ECC_BCH60_1K, 60), +}; + +static int meson_nand_calc_ecc_bytes(int step_size, int strength) +{ + int ecc_bytes; + + if (step_size == 512 && strength == 8) + return ECC_PARITY_BCH8_512B; + + ecc_bytes = DIV_ROUND_UP(strength * fls(step_size * 8), 8); + ecc_bytes = ALIGN(ecc_bytes, 2); + + return ecc_bytes; +} + +NAND_ECC_CAPS_SINGLE(meson_gxl_ecc_caps, + meson_nand_calc_ecc_bytes, 1024, 8, 24, 30, 40, 50, 60); +NAND_ECC_CAPS_SINGLE(meson_axg_ecc_caps, + meson_nand_calc_ecc_bytes, 1024, 8); + +static struct meson_nfc_nand_chip *to_meson_nand(struct nand_chip *nand) +{ + return container_of(nand, struct meson_nfc_nand_chip, nand); +} + +static void meson_nfc_select_chip(struct nand_chip *nand, int chip) +{ + struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); + struct meson_nfc *nfc = nand_get_controller_data(nand); + int ret, value; + + if (chip < 0 || WARN_ON_ONCE(chip >= meson_chip->nsels)) + return; + + nfc->param.chip_select = meson_chip->sels[chip] ? NAND_CE1 : NAND_CE0; + nfc->param.rb_select = nfc->param.chip_select; + nfc->timing.twb = meson_chip->twb; + nfc->timing.tadl = meson_chip->tadl; + nfc->timing.tbers_max = meson_chip->tbers_max; + + if (nfc->clk_rate != meson_chip->clk_rate) { + ret = clk_set_rate(nfc->device_clk, meson_chip->clk_rate); + if (ret) { + dev_err(nfc->dev, "failed to set clock rate\n"); + return; + } + nfc->clk_rate = meson_chip->clk_rate; + } + if (nfc->bus_timing != meson_chip->bus_timing) { + value = (NFC_CLK_CYCLE - 1) | (meson_chip->bus_timing << 5); + writel(value, nfc->reg_base + NFC_REG_CFG); + writel((1 << 31), nfc->reg_base + NFC_REG_CMD); + nfc->bus_timing = meson_chip->bus_timing; + } +} + +static void meson_nfc_cmd_idle(struct meson_nfc *nfc, u32 time) +{ + writel(nfc->param.chip_select | NFC_CMD_IDLE | (time & 0x3ff), + nfc->reg_base + NFC_REG_CMD); +} + +static void meson_nfc_cmd_seed(struct meson_nfc *nfc, u32 seed) +{ + writel(NFC_CMD_SEED | (0xc2 + (seed & 0x7fff)), + nfc->reg_base + NFC_REG_CMD); +} + +static void meson_nfc_cmd_access(struct nand_chip *nand, int raw, bool dir, + int scrambler) +{ + struct mtd_info *mtd = nand_to_mtd(nand); + struct meson_nfc *nfc = nand_get_controller_data(mtd_to_nand(mtd)); + struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); + u32 bch = meson_chip->bch_mode, cmd; + int len = mtd->writesize, pagesize, pages; + + pagesize = nand->ecc.size; + + if (raw) { + len = mtd->writesize + mtd->oobsize; + cmd = (len & GENMASK(5, 0)) | scrambler | DMA_DIR(dir); + writel(cmd, nfc->reg_base + NFC_REG_CMD); + return; + } + + pages = len / nand->ecc.size; + + cmd = CMDRWGEN(DMA_DIR(dir), scrambler, bch, + NFC_CMD_SHORTMODE_DISABLE, pagesize, pages); + + writel(cmd, nfc->reg_base + NFC_REG_CMD); +} + +static void meson_nfc_drain_cmd(struct meson_nfc *nfc) +{ + /* + * Insert two commands to make sure all valid commands are finished. + * + * The Nand flash controller is designed as two stages pipleline - + * a) fetch and b) excute. + * There might be cases when the driver see command queue is empty, + * but the Nand flash controller still has two commands buffered, + * one is fetched into NFC request queue (ready to run), and another + * is actively executing. So pushing 2 "IDLE" commands guarantees that + * the pipeline is emptied. + */ + meson_nfc_cmd_idle(nfc, 0); + meson_nfc_cmd_idle(nfc, 0); +} + +static int meson_nfc_wait_cmd_finish(struct meson_nfc *nfc, + unsigned int timeout_ms) +{ + u32 cmd_size = 0; + int ret; + + /* wait cmd fifo is empty */ + ret = readl_relaxed_poll_timeout(nfc->reg_base + NFC_REG_CMD, cmd_size, + !NFC_CMD_GET_SIZE(cmd_size), + 10, timeout_ms * 1000); + if (ret) + dev_err(nfc->dev, "wait for empty CMD FIFO time out\n"); + + return ret; +} + +static int meson_nfc_wait_dma_finish(struct meson_nfc *nfc) +{ + meson_nfc_drain_cmd(nfc); + + return meson_nfc_wait_cmd_finish(nfc, DMA_BUSY_TIMEOUT); +} + +static u8 *meson_nfc_oob_ptr(struct nand_chip *nand, int i) +{ + struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); + int len; + + len = nand->ecc.size * (i + 1) + (nand->ecc.bytes + 2) * i; + + return meson_chip->data_buf + len; +} + +static u8 *meson_nfc_data_ptr(struct nand_chip *nand, int i) +{ + struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); + int len, temp; + + temp = nand->ecc.size + nand->ecc.bytes; + len = (temp + 2) * i; + + return meson_chip->data_buf + len; +} + +static void meson_nfc_get_data_oob(struct nand_chip *nand, + u8 *buf, u8 *oobbuf) +{ + int i, oob_len = 0; + u8 *dsrc, *osrc; + + oob_len = nand->ecc.bytes + 2; + for (i = 0; i < nand->ecc.steps; i++) { + if (buf) { + dsrc = meson_nfc_data_ptr(nand, i); + memcpy(buf, dsrc, nand->ecc.size); + buf += nand->ecc.size; + } + osrc = meson_nfc_oob_ptr(nand, i); + memcpy(oobbuf, osrc, oob_len); + oobbuf += oob_len; + } +} + +static void meson_nfc_set_data_oob(struct nand_chip *nand, + const u8 *buf, u8 *oobbuf) +{ + int i, oob_len = 0; + u8 *dsrc, *osrc; + + oob_len = nand->ecc.bytes + 2; + for (i = 0; i < nand->ecc.steps; i++) { + if (buf) { + dsrc = meson_nfc_data_ptr(nand, i); + memcpy(dsrc, buf, nand->ecc.size); + buf += nand->ecc.size; + } + osrc = meson_nfc_oob_ptr(nand, i); + memcpy(osrc, oobbuf, oob_len); + oobbuf += oob_len; + } +} + +static int meson_nfc_queue_rb(struct meson_nfc *nfc, int timeout_ms) +{ + u32 cmd, cfg; + int ret = 0; + + meson_nfc_cmd_idle(nfc, nfc->timing.twb); + meson_nfc_drain_cmd(nfc); + meson_nfc_wait_cmd_finish(nfc, CMD_FIFO_EMPTY_TIMEOUT); + + cfg = readl(nfc->reg_base + NFC_REG_CFG); + cfg |= NFC_RB_IRQ_EN; + writel(cfg, nfc->reg_base + NFC_REG_CFG); + + init_completion(&nfc->completion); + + /* use the max erase time as the maximum clock for waiting R/B */ + cmd = NFC_CMD_RB | NFC_CMD_RB_INT + | nfc->param.chip_select | nfc->timing.tbers_max; + writel(cmd, nfc->reg_base + NFC_REG_CMD); + + ret = wait_for_completion_timeout(&nfc->completion, + msecs_to_jiffies(timeout_ms)); + if (ret == 0) + ret = -1; + + return ret; +} + +static void meson_nfc_set_user_byte(struct nand_chip *nand, u8 *oob_buf) +{ + struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); + __le64 *info; + int i, count; + + for (i = 0, count = 0; i < nand->ecc.steps; i++, count += 2) { + info = &meson_chip->info_buf[i]; + *info |= oob_buf[count]; + *info |= oob_buf[count + 1] << 8; + } +} + +static void meson_nfc_get_user_byte(struct nand_chip *nand, u8 *oob_buf) +{ + struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); + __le64 *info; + int i, count; + + for (i = 0, count = 0; i < nand->ecc.steps; i++, count += 2) { + info = &meson_chip->info_buf[i]; + oob_buf[count] = *info; + oob_buf[count + 1] = *info >> 8; + } +} + +static int meson_nfc_ecc_correct(struct nand_chip *nand, u32 *bitflips, + u64 *correct_bitmap) +{ + struct mtd_info *mtd = nand_to_mtd(nand); + struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); + __le64 *info; + int ret = 0, i; + + for (i = 0; i < nand->ecc.steps; i++) { + info = &meson_chip->info_buf[i]; + if (ECC_ERR_CNT(*info) != ECC_UNCORRECTABLE) { + mtd->ecc_stats.corrected += ECC_ERR_CNT(*info); + *bitflips = max_t(u32, *bitflips, ECC_ERR_CNT(*info)); + *correct_bitmap |= 1 >> i; + continue; + } + if ((nand->options & NAND_NEED_SCRAMBLING) && + ECC_ZERO_CNT(*info) < nand->ecc.strength) { + mtd->ecc_stats.corrected += ECC_ZERO_CNT(*info); + *bitflips = max_t(u32, *bitflips, + ECC_ZERO_CNT(*info)); + ret = ECC_CHECK_RETURN_FF; + } else { + ret = -EBADMSG; + } + } + return ret; +} + +static int meson_nfc_dma_buffer_setup(struct nand_chip *nand, u8 *databuf, + int datalen, u8 *infobuf, int infolen, + enum dma_data_direction dir) +{ + struct meson_nfc *nfc = nand_get_controller_data(nand); + u32 cmd; + int ret = 0; + + nfc->daddr = dma_map_single(nfc->dev, (void *)databuf, datalen, dir); + ret = dma_mapping_error(nfc->dev, nfc->daddr); + if (ret) { + dev_err(nfc->dev, "DMA mapping error\n"); + return ret; + } + cmd = GENCMDDADDRL(NFC_CMD_ADL, nfc->daddr); + writel(cmd, nfc->reg_base + NFC_REG_CMD); + + cmd = GENCMDDADDRH(NFC_CMD_ADH, nfc->daddr); + writel(cmd, nfc->reg_base + NFC_REG_CMD); + + if (infobuf) { + nfc->iaddr = dma_map_single(nfc->dev, infobuf, infolen, dir); + ret = dma_mapping_error(nfc->dev, nfc->iaddr); + if (ret) { + dev_err(nfc->dev, "DMA mapping error\n"); + dma_unmap_single(nfc->dev, + nfc->daddr, datalen, dir); + return ret; + } + cmd = GENCMDIADDRL(NFC_CMD_AIL, nfc->iaddr); + writel(cmd, nfc->reg_base + NFC_REG_CMD); + + cmd = GENCMDIADDRH(NFC_CMD_AIH, nfc->iaddr); + writel(cmd, nfc->reg_base + NFC_REG_CMD); + } + + return ret; +} + +static void meson_nfc_dma_buffer_release(struct nand_chip *nand, + int infolen, int datalen, + enum dma_data_direction dir) +{ + struct meson_nfc *nfc = nand_get_controller_data(nand); + + dma_unmap_single(nfc->dev, nfc->daddr, datalen, dir); + if (infolen) + dma_unmap_single(nfc->dev, nfc->iaddr, infolen, dir); +} + +static int meson_nfc_read_buf(struct nand_chip *nand, u8 *buf, int len) +{ + struct meson_nfc *nfc = nand_get_controller_data(nand); + int ret = 0; + u32 cmd; + u8 *info; + + info = kzalloc(PER_INFO_BYTE, GFP_KERNEL); + ret = meson_nfc_dma_buffer_setup(nand, buf, len, info, + PER_INFO_BYTE, DMA_FROM_DEVICE); + if (ret) + return ret; + + cmd = NFC_CMD_N2M | (len & GENMASK(5, 0)); + writel(cmd, nfc->reg_base + NFC_REG_CMD); + + meson_nfc_drain_cmd(nfc); + meson_nfc_wait_cmd_finish(nfc, 1000); + meson_nfc_dma_buffer_release(nand, len, PER_INFO_BYTE, DMA_FROM_DEVICE); + kfree(info); + + return ret; +} + +static int meson_nfc_write_buf(struct nand_chip *nand, u8 *buf, int len) +{ + struct meson_nfc *nfc = nand_get_controller_data(nand); + int ret = 0; + u32 cmd; + + ret = meson_nfc_dma_buffer_setup(nand, buf, len, NULL, + 0, DMA_TO_DEVICE); + if (ret) + return ret; + + cmd = NFC_CMD_M2N | (len & GENMASK(5, 0)); + writel(cmd, nfc->reg_base + NFC_REG_CMD); + + meson_nfc_drain_cmd(nfc); + meson_nfc_wait_cmd_finish(nfc, 1000); + meson_nfc_dma_buffer_release(nand, len, 0, DMA_TO_DEVICE); + + return ret; +} + +static int meson_nfc_rw_cmd_prepare_and_execute(struct nand_chip *nand, + int page, bool in) +{ + struct mtd_info *mtd = nand_to_mtd(nand); + struct meson_nfc *nfc = nand_get_controller_data(nand); + const struct nand_sdr_timings *sdr = + nand_get_sdr_timings(&nand->data_interface); + u32 *addrs = nfc->cmdfifo.rw.addrs; + u32 cs = nfc->param.chip_select; + u32 cmd0, cmd_num, row_start; + int ret = 0, i; + + cmd_num = sizeof(struct nand_rw_cmd) / sizeof(int); + + cmd0 = in ? NAND_CMD_READ0 : NAND_CMD_SEQIN; + nfc->cmdfifo.rw.cmd0 = cs | NFC_CMD_CLE | cmd0; + + addrs[0] = cs | NFC_CMD_ALE | 0; + if (mtd->writesize <= 512) { + cmd_num--; + row_start = 1; + } else { + addrs[1] = cs | NFC_CMD_ALE | 0; + row_start = 2; + } + + addrs[row_start] = cs | NFC_CMD_ALE | ROW_ADDER(page, 0); + addrs[row_start + 1] = cs | NFC_CMD_ALE | ROW_ADDER(page, 1); + + if (nand->options & NAND_ROW_ADDR_3) + addrs[row_start + 2] = + cs | NFC_CMD_ALE | ROW_ADDER(page, 2); + else + cmd_num--; + + /* subtract cmd1 */ + cmd_num--; + + for (i = 0; i < cmd_num; i++) + writel_relaxed(nfc->cmdfifo.cmd[i], + nfc->reg_base + NFC_REG_CMD); + + if (in) { + nfc->cmdfifo.rw.cmd1 = cs | NFC_CMD_CLE | NAND_CMD_READSTART; + writel(nfc->cmdfifo.rw.cmd1, nfc->reg_base + NFC_REG_CMD); + meson_nfc_queue_rb(nfc, PSEC_TO_MSEC(sdr->tR_max)); + } else { + meson_nfc_cmd_idle(nfc, nfc->timing.tadl); + } + + return ret; +} + +static int meson_nfc_write_page_sub(struct nand_chip *nand, + int page, int raw) +{ + struct mtd_info *mtd = nand_to_mtd(nand); + const struct nand_sdr_timings *sdr = + nand_get_sdr_timings(&nand->data_interface); + struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); + struct meson_nfc *nfc = nand_get_controller_data(nand); + int data_len, info_len; + u32 cmd; + int ret; + + meson_nfc_select_chip(nand, nand->cur_cs); + + data_len = mtd->writesize + mtd->oobsize; + info_len = nand->ecc.steps * PER_INFO_BYTE; + + ret = meson_nfc_rw_cmd_prepare_and_execute(nand, page, DIRWRITE); + if (ret) + return ret; + + ret = meson_nfc_dma_buffer_setup(nand, meson_chip->data_buf, + data_len, (u8 *)meson_chip->info_buf, + info_len, DMA_TO_DEVICE); + if (ret) + return ret; + + if (nand->options & NAND_NEED_SCRAMBLING) { + meson_nfc_cmd_seed(nfc, page); + meson_nfc_cmd_access(nand, raw, DIRWRITE, + NFC_CMD_SCRAMBLER_ENABLE); + } else { + meson_nfc_cmd_access(nand, raw, DIRWRITE, + NFC_CMD_SCRAMBLER_DISABLE); + } + + cmd = nfc->param.chip_select | NFC_CMD_CLE | NAND_CMD_PAGEPROG; + writel(cmd, nfc->reg_base + NFC_REG_CMD); + meson_nfc_queue_rb(nfc, PSEC_TO_MSEC(sdr->tPROG_max)); + + meson_nfc_dma_buffer_release(nand, data_len, info_len, DMA_TO_DEVICE); + + return ret; +} + +static int meson_nfc_write_page_raw(struct nand_chip *nand, const u8 *buf, + int oob_required, int page) +{ + u8 *oob_buf = nand->oob_poi; + + meson_nfc_set_data_oob(nand, buf, oob_buf); + + return meson_nfc_write_page_sub(nand, page, 1); +} + +static int meson_nfc_write_page_hwecc(struct nand_chip *nand, + const u8 *buf, int oob_required, int page) +{ + struct mtd_info *mtd = nand_to_mtd(nand); + struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); + u8 *oob_buf = nand->oob_poi; + + memcpy(meson_chip->data_buf, buf, mtd->writesize); + memset(meson_chip->info_buf, 0, nand->ecc.steps * PER_INFO_BYTE); + meson_nfc_set_user_byte(nand, oob_buf); + + return meson_nfc_write_page_sub(nand, page, 0); +} + +static void meson_nfc_check_ecc_pages_valid(struct meson_nfc *nfc, + struct nand_chip *nand, int raw) +{ + struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); + __le64 *info; + u32 neccpages; + int ret; + + neccpages = raw ? 1 : nand->ecc.steps; + info = &meson_chip->info_buf[neccpages - 1]; + do { + usleep_range(10, 15); + /* info is updated by nfc dma engine*/ + smp_rmb(); + ret = *info & ECC_COMPLETE; + } while (!ret); +} + +static int meson_nfc_read_page_sub(struct nand_chip *nand, + int page, int raw) +{ + struct mtd_info *mtd = nand_to_mtd(nand); + struct meson_nfc *nfc = nand_get_controller_data(nand); + struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); + int data_len, info_len; + int ret; + + meson_nfc_select_chip(nand, nand->cur_cs); + + data_len = mtd->writesize + mtd->oobsize; + info_len = nand->ecc.steps * PER_INFO_BYTE; + + ret = meson_nfc_rw_cmd_prepare_and_execute(nand, page, DIRREAD); + if (ret) + return ret; + + ret = meson_nfc_dma_buffer_setup(nand, meson_chip->data_buf, + data_len, (u8 *)meson_chip->info_buf, + info_len, DMA_FROM_DEVICE); + if (ret) + return ret; + + if (nand->options & NAND_NEED_SCRAMBLING) { + meson_nfc_cmd_seed(nfc, page); + meson_nfc_cmd_access(nand, raw, DIRREAD, + NFC_CMD_SCRAMBLER_ENABLE); + } else { + meson_nfc_cmd_access(nand, raw, DIRREAD, + NFC_CMD_SCRAMBLER_DISABLE); + } + + ret = meson_nfc_wait_dma_finish(nfc); + meson_nfc_check_ecc_pages_valid(nfc, nand, raw); + + meson_nfc_dma_buffer_release(nand, data_len, info_len, DMA_FROM_DEVICE); + + return ret; +} + +static int meson_nfc_read_page_raw(struct nand_chip *nand, u8 *buf, + int oob_required, int page) +{ + u8 *oob_buf = nand->oob_poi; + int ret; + + ret = meson_nfc_read_page_sub(nand, page, 1); + if (ret) + return ret; + + meson_nfc_get_data_oob(nand, buf, oob_buf); + + return 0; +} + +static int meson_nfc_read_page_hwecc(struct nand_chip *nand, u8 *buf, + int oob_required, int page) +{ + struct mtd_info *mtd = nand_to_mtd(nand); + struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); + struct nand_ecc_ctrl *ecc = &nand->ecc; + u64 correct_bitmap = 0; + u32 bitflips = 0; + u8 *oob_buf = nand->oob_poi; + int ret, i; + + ret = meson_nfc_read_page_sub(nand, page, 0); + if (ret) + return ret; + + meson_nfc_get_user_byte(nand, oob_buf); + ret = meson_nfc_ecc_correct(nand, &bitflips, &correct_bitmap); + if (ret == ECC_CHECK_RETURN_FF) { + if (buf) + memset(buf, 0xff, mtd->writesize); + memset(oob_buf, 0xff, mtd->oobsize); + } else if (ret < 0) { + if ((nand->options & NAND_NEED_SCRAMBLING) || !buf) { + mtd->ecc_stats.failed++; + return bitflips; + } + ret = meson_nfc_read_page_raw(nand, buf, 0, page); + if (ret) + return ret; + + for (i = 0; i < nand->ecc.steps ; i++) { + u8 *data = buf + i * ecc->size; + u8 *oob = nand->oob_poi + i * (ecc->bytes + 2); + + if (correct_bitmap & (1 << i)) + continue; + ret = nand_check_erased_ecc_chunk(data, ecc->size, + oob, ecc->bytes + 2, + NULL, 0, + ecc->strength); + if (ret < 0) { + mtd->ecc_stats.failed++; + } else { + mtd->ecc_stats.corrected += ret; + bitflips = max_t(u32, bitflips, ret); + } + } + } else if (buf && buf != meson_chip->data_buf) { + memcpy(buf, meson_chip->data_buf, mtd->writesize); + } + + return bitflips; +} + +static int meson_nfc_read_oob_raw(struct nand_chip *nand, int page) +{ + return meson_nfc_read_page_raw(nand, NULL, 1, page); +} + +static int meson_nfc_read_oob(struct nand_chip *nand, int page) +{ + return meson_nfc_read_page_hwecc(nand, NULL, 1, page); +} + +static bool meson_nfc_is_buffer_dma_safe(const void *buffer) +{ + if (virt_addr_valid(buffer) && (!object_is_on_stack(buffer))) + return true; + return false; +} + +static void * +meson_nand_op_get_dma_safe_input_buf(const struct nand_op_instr *instr) +{ + if (WARN_ON(instr->type != NAND_OP_DATA_IN_INSTR)) + return NULL; + + if (meson_nfc_is_buffer_dma_safe(instr->ctx.data.buf.in)) + return instr->ctx.data.buf.in; + + return kzalloc(instr->ctx.data.len, GFP_KERNEL); +} + +static void +meson_nand_op_put_dma_safe_input_buf(const struct nand_op_instr *instr, + void *buf) +{ + if (WARN_ON(instr->type != NAND_OP_DATA_IN_INSTR) || + WARN_ON(!buf)) + return; + + if (buf == instr->ctx.data.buf.in) + return; + + memcpy(instr->ctx.data.buf.in, buf, instr->ctx.data.len); + kfree(buf); +} + +static void * +meson_nand_op_get_dma_safe_output_buf(const struct nand_op_instr *instr) +{ + if (WARN_ON(instr->type != NAND_OP_DATA_OUT_INSTR)) + return NULL; + + if (meson_nfc_is_buffer_dma_safe(instr->ctx.data.buf.out)) + return (void *)instr->ctx.data.buf.out; + + return kmemdup(instr->ctx.data.buf.out, + instr->ctx.data.len, GFP_KERNEL); +} + +static void +meson_nand_op_put_dma_safe_output_buf(const struct nand_op_instr *instr, + const void *buf) +{ + if (WARN_ON(instr->type != NAND_OP_DATA_OUT_INSTR) || + WARN_ON(!buf)) + return; + + if (buf != instr->ctx.data.buf.out) + kfree(buf); +} + +static int meson_nfc_exec_op(struct nand_chip *nand, + const struct nand_operation *op, bool check_only) +{ + struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); + struct meson_nfc *nfc = nand_get_controller_data(nand); + const struct nand_op_instr *instr = NULL; + void *buf; + u32 op_id, delay_idle, cmd; + int i; + + meson_nfc_select_chip(nand, op->cs); + for (op_id = 0; op_id < op->ninstrs; op_id++) { + instr = &op->instrs[op_id]; + delay_idle = DIV_ROUND_UP(PSEC_TO_NSEC(instr->delay_ns), + meson_chip->level1_divider * + NFC_CLK_CYCLE); + switch (instr->type) { + case NAND_OP_CMD_INSTR: + cmd = nfc->param.chip_select | NFC_CMD_CLE; + cmd |= instr->ctx.cmd.opcode & 0xff; + writel(cmd, nfc->reg_base + NFC_REG_CMD); + meson_nfc_cmd_idle(nfc, delay_idle); + break; + + case NAND_OP_ADDR_INSTR: + for (i = 0; i < instr->ctx.addr.naddrs; i++) { + cmd = nfc->param.chip_select | NFC_CMD_ALE; + cmd |= instr->ctx.addr.addrs[i] & 0xff; + writel(cmd, nfc->reg_base + NFC_REG_CMD); + } + meson_nfc_cmd_idle(nfc, delay_idle); + break; + + case NAND_OP_DATA_IN_INSTR: + buf = meson_nand_op_get_dma_safe_input_buf(instr); + if (!buf) + return -ENOMEM; + meson_nfc_read_buf(nand, buf, instr->ctx.data.len); + meson_nand_op_put_dma_safe_input_buf(instr, buf); + break; + + case NAND_OP_DATA_OUT_INSTR: + buf = meson_nand_op_get_dma_safe_output_buf(instr); + if (!buf) + return -ENOMEM; + meson_nfc_write_buf(nand, buf, instr->ctx.data.len); + meson_nand_op_put_dma_safe_output_buf(instr, buf); + break; + + case NAND_OP_WAITRDY_INSTR: + meson_nfc_queue_rb(nfc, instr->ctx.waitrdy.timeout_ms); + if (instr->delay_ns) + meson_nfc_cmd_idle(nfc, delay_idle); + break; + } + } + meson_nfc_wait_cmd_finish(nfc, 1000); + return 0; +} + +static int meson_ooblayout_ecc(struct mtd_info *mtd, int section, + struct mtd_oob_region *oobregion) +{ + struct nand_chip *nand = mtd_to_nand(mtd); + + if (section >= nand->ecc.steps) + return -ERANGE; + + oobregion->offset = 2 + (section * (2 + nand->ecc.bytes)); + oobregion->length = nand->ecc.bytes; + + return 0; +} + +static int meson_ooblayout_free(struct mtd_info *mtd, int section, + struct mtd_oob_region *oobregion) +{ + struct nand_chip *nand = mtd_to_nand(mtd); + + if (section >= nand->ecc.steps) + return -ERANGE; + + oobregion->offset = section * (2 + nand->ecc.bytes); + oobregion->length = 2; + + return 0; +} + +static const struct mtd_ooblayout_ops meson_ooblayout_ops = { + .ecc = meson_ooblayout_ecc, + .free = meson_ooblayout_free, +}; + +static int meson_nfc_clk_init(struct meson_nfc *nfc) +{ + int ret; + + /* request core clock */ + nfc->core_clk = devm_clk_get(nfc->dev, "core"); + if (IS_ERR(nfc->core_clk)) { + dev_err(nfc->dev, "failed to get core clock\n"); + return PTR_ERR(nfc->core_clk); + } + + nfc->device_clk = devm_clk_get(nfc->dev, "device"); + if (IS_ERR(nfc->device_clk)) { + dev_err(nfc->dev, "failed to get device clock\n"); + return PTR_ERR(nfc->device_clk); + } + + nfc->phase_tx = devm_clk_get(nfc->dev, "tx"); + if (IS_ERR(nfc->phase_tx)) { + dev_err(nfc->dev, "failed to get TX clk\n"); + return PTR_ERR(nfc->phase_tx); + } + + nfc->phase_rx = devm_clk_get(nfc->dev, "rx"); + if (IS_ERR(nfc->phase_rx)) { + dev_err(nfc->dev, "failed to get RX clk\n"); + return PTR_ERR(nfc->phase_rx); + } + + /* init SD_EMMC_CLOCK to sane defaults w/min clock rate */ + regmap_update_bits(nfc->reg_clk, + 0, CLK_SELECT_NAND, CLK_SELECT_NAND); + + ret = clk_prepare_enable(nfc->core_clk); + if (ret) { + dev_err(nfc->dev, "failed to enable core clock\n"); + return ret; + } + + ret = clk_prepare_enable(nfc->device_clk); + if (ret) { + dev_err(nfc->dev, "failed to enable device clock\n"); + goto err_device_clk; + } + + ret = clk_prepare_enable(nfc->phase_tx); + if (ret) { + dev_err(nfc->dev, "failed to enable TX clock\n"); + goto err_phase_tx; + } + + ret = clk_prepare_enable(nfc->phase_rx); + if (ret) { + dev_err(nfc->dev, "failed to enable RX clock\n"); + goto err_phase_rx; + } + + ret = clk_set_rate(nfc->device_clk, 24000000); + if (ret) + goto err_phase_rx; + + return 0; +err_phase_rx: + clk_disable_unprepare(nfc->phase_tx); +err_phase_tx: + clk_disable_unprepare(nfc->device_clk); +err_device_clk: + clk_disable_unprepare(nfc->core_clk); + return ret; +} + +static void meson_nfc_disable_clk(struct meson_nfc *nfc) +{ + clk_disable_unprepare(nfc->phase_rx); + clk_disable_unprepare(nfc->phase_tx); + clk_disable_unprepare(nfc->device_clk); + clk_disable_unprepare(nfc->core_clk); +} + +static void meson_nfc_free_buffer(struct nand_chip *nand) +{ + struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); + + kfree(meson_chip->info_buf); + kfree(meson_chip->data_buf); +} + +static int meson_chip_buffer_init(struct nand_chip *nand) +{ + struct mtd_info *mtd = nand_to_mtd(nand); + struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); + u32 page_bytes, info_bytes, nsectors; + + nsectors = mtd->writesize / nand->ecc.size; + + page_bytes = mtd->writesize + mtd->oobsize; + info_bytes = nsectors * PER_INFO_BYTE; + + meson_chip->data_buf = kmalloc(page_bytes, GFP_KERNEL); + if (!meson_chip->data_buf) + return -ENOMEM; + + meson_chip->info_buf = kmalloc(info_bytes, GFP_KERNEL); + if (!meson_chip->info_buf) { + kfree(meson_chip->data_buf); + return -ENOMEM; + } + + return 0; +} + +static +int meson_nfc_setup_data_interface(struct nand_chip *nand, int csline, + const struct nand_data_interface *conf) +{ + struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); + const struct nand_sdr_timings *timings; + u32 div, bt_min, bt_max, tbers_clocks; + + timings = nand_get_sdr_timings(conf); + if (IS_ERR(timings)) + return -ENOTSUPP; + + if (csline == NAND_DATA_IFACE_CHECK_ONLY) + return 0; + + div = DIV_ROUND_UP((timings->tRC_min / 1000), NFC_CLK_CYCLE); + bt_min = (timings->tREA_max + NFC_DEFAULT_DELAY) / div; + bt_max = (NFC_DEFAULT_DELAY + timings->tRHOH_min + + timings->tRC_min / 2) / div; + + meson_chip->twb = DIV_ROUND_UP(PSEC_TO_NSEC(timings->tWB_max), + div * NFC_CLK_CYCLE); + meson_chip->tadl = DIV_ROUND_UP(PSEC_TO_NSEC(timings->tADL_min), + div * NFC_CLK_CYCLE); + tbers_clocks = DIV_ROUND_UP_ULL(PSEC_TO_NSEC(timings->tBERS_max), + div * NFC_CLK_CYCLE); + meson_chip->tbers_max = ilog2(tbers_clocks); + if (!is_power_of_2(tbers_clocks)) + meson_chip->tbers_max++; + + bt_min = DIV_ROUND_UP(bt_min, 1000); + bt_max = DIV_ROUND_UP(bt_max, 1000); + + if (bt_max < bt_min) + return -EINVAL; + + meson_chip->level1_divider = div; + meson_chip->clk_rate = 1000000000 / meson_chip->level1_divider; + meson_chip->bus_timing = (bt_min + bt_max) / 2 + 1; + + return 0; +} + +static int meson_nand_bch_mode(struct nand_chip *nand) +{ + struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); + int i; + + if (nand->ecc.strength > 60 || nand->ecc.strength < 8) + return -EINVAL; + + for (i = 0; i < ARRAY_SIZE(meson_ecc); i++) { + if (meson_ecc[i].strength == nand->ecc.strength) { + meson_chip->bch_mode = meson_ecc[i].bch; + return 0; + } + } + + return -EINVAL; +} + +static void meson_nand_detach_chip(struct nand_chip *nand) +{ + meson_nfc_free_buffer(nand); +} + +static int meson_nand_attach_chip(struct nand_chip *nand) +{ + struct meson_nfc *nfc = nand_get_controller_data(nand); + struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand); + struct mtd_info *mtd = nand_to_mtd(nand); + int nsectors = mtd->writesize / 1024; + int ret; + + if (!mtd->name) { + mtd->name = devm_kasprintf(nfc->dev, GFP_KERNEL, + "%s:nand%d", + dev_name(nfc->dev), + meson_chip->sels[0]); + if (!mtd->name) + return -ENOMEM; + } + + if (nand->bbt_options & NAND_BBT_USE_FLASH) + nand->bbt_options |= NAND_BBT_NO_OOB; + + nand->options |= NAND_NO_SUBPAGE_WRITE; + + ret = nand_ecc_choose_conf(nand, nfc->data->ecc_caps, + mtd->oobsize - 2 * nsectors); + if (ret) { + dev_err(nfc->dev, "failed to ECC init\n"); + return -EINVAL; + } + + ret = meson_nand_bch_mode(nand); + if (ret) + return -EINVAL; + + nand->ecc.mode = NAND_ECC_HW; + nand->ecc.write_page_raw = meson_nfc_write_page_raw; + nand->ecc.write_page = meson_nfc_write_page_hwecc; + nand->ecc.write_oob_raw = nand_write_oob_std; + nand->ecc.write_oob = nand_write_oob_std; + + nand->ecc.read_page_raw = meson_nfc_read_page_raw; + nand->ecc.read_page = meson_nfc_read_page_hwecc; + nand->ecc.read_oob_raw = meson_nfc_read_oob_raw; + nand->ecc.read_oob = meson_nfc_read_oob; + + if (nand->options & NAND_BUSWIDTH_16) { + dev_err(nfc->dev, "16bits bus width not supported"); + return -EINVAL; + } + ret = meson_chip_buffer_init(nand); + if (ret) + return -ENOMEM; + + return ret; +} + +static const struct nand_controller_ops meson_nand_controller_ops = { + .attach_chip = meson_nand_attach_chip, + .detach_chip = meson_nand_detach_chip, + .setup_data_interface = meson_nfc_setup_data_interface, + .exec_op = meson_nfc_exec_op, +}; + +static int +meson_nfc_nand_chip_init(struct device *dev, + struct meson_nfc *nfc, struct device_node *np) +{ + struct meson_nfc_nand_chip *meson_chip; + struct nand_chip *nand; + struct mtd_info *mtd; + int ret, i; + u32 tmp, nsels; + + if (!of_get_property(np, "reg", &nsels)) + return -EINVAL; + + nsels /= sizeof(u32); + if (!nsels || nsels > MAX_CE_NUM) { + dev_err(dev, "invalid register property size\n"); + return -EINVAL; + } + + meson_chip = devm_kzalloc(dev, + sizeof(*meson_chip) + (nsels * sizeof(u8)), + GFP_KERNEL); + if (!meson_chip) + return -ENOMEM; + + meson_chip->nsels = nsels; + + for (i = 0; i < nsels; i++) { + ret = of_property_read_u32_index(np, "reg", i, &tmp); + if (ret) { + dev_err(dev, "could not retrieve register property: %d\n", + ret); + return ret; + } + + if (test_and_set_bit(tmp, &nfc->assigned_cs)) { + dev_err(dev, "CS %d already assigned\n", tmp); + return -EINVAL; + } + } + + nand = &meson_chip->nand; + nand->controller = &nfc->controller; + nand->controller->ops = &meson_nand_controller_ops; + nand_set_flash_node(nand, np); + nand_set_controller_data(nand, nfc); + + nand->options |= NAND_USE_BOUNCE_BUFFER; + mtd = nand_to_mtd(nand); + mtd->owner = THIS_MODULE; + mtd->dev.parent = dev; + + ret = nand_scan(nand, nsels); + if (ret) + return ret; + + ret = mtd_device_register(mtd, NULL, 0); + if (ret) { + dev_err(dev, "failed to register MTD device: %d\n", ret); + nand_cleanup(nand); + return ret; + } + + list_add_tail(&meson_chip->node, &nfc->chips); + + return 0; +} + +static int meson_nfc_nand_chip_cleanup(struct meson_nfc *nfc) +{ + struct meson_nfc_nand_chip *meson_chip; + struct mtd_info *mtd; + int ret; + + while (!list_empty(&nfc->chips)) { + meson_chip = list_first_entry(&nfc->chips, + struct meson_nfc_nand_chip, node); + mtd = nand_to_mtd(&meson_chip->nand); + ret = mtd_device_unregister(mtd); + if (ret) + return ret; + + meson_nfc_free_buffer(&meson_chip->nand); + nand_cleanup(&meson_chip->nand); + list_del(&meson_chip->node); + } + + return 0; +} + +static int meson_nfc_nand_chips_init(struct device *dev, + struct meson_nfc *nfc) +{ + struct device_node *np = dev->of_node; + struct device_node *nand_np; + int ret; + + for_each_child_of_node(np, nand_np) { + ret = meson_nfc_nand_chip_init(dev, nfc, nand_np); + if (ret) { + meson_nfc_nand_chip_cleanup(nfc); + return ret; + } + } + + return 0; +} + +static irqreturn_t meson_nfc_irq(int irq, void *id) +{ + struct meson_nfc *nfc = id; + u32 cfg; + + cfg = readl(nfc->reg_base + NFC_REG_CFG); + if (!(cfg & NFC_RB_IRQ_EN)) + return IRQ_NONE; + + cfg &= ~(NFC_RB_IRQ_EN); + writel(cfg, nfc->reg_base + NFC_REG_CFG); + + complete(&nfc->completion); + return IRQ_HANDLED; +} + +static const struct meson_nfc_data meson_gxl_data = { + .ecc_caps = &meson_gxl_ecc_caps, +}; + +static const struct meson_nfc_data meson_axg_data = { + .ecc_caps = &meson_axg_ecc_caps, +}; + +static const struct of_device_id meson_nfc_id_table[] = { + { + .compatible = "amlogic,meson-gxl-nfc", + .data = &meson_gxl_data, + }, { + .compatible = "amlogic,meson-axg-nfc", + .data = &meson_axg_data, + }, + {} +}; +MODULE_DEVICE_TABLE(of, meson_nfc_id_table); + +static int meson_nfc_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct meson_nfc *nfc; + struct resource *res; + int ret, irq; + + nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL); + if (!nfc) + return -ENOMEM; + + nfc->data = of_device_get_match_data(&pdev->dev); + if (!nfc->data) + return -ENODEV; + + nand_controller_init(&nfc->controller); + INIT_LIST_HEAD(&nfc->chips); + + nfc->dev = dev; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + nfc->reg_base = devm_ioremap_resource(dev, res); + if (IS_ERR(nfc->reg_base)) + return PTR_ERR(nfc->reg_base); + + nfc->reg_clk = + syscon_regmap_lookup_by_phandle(dev->of_node, + "amlogic,mmc-syscon"); + if (IS_ERR(nfc->reg_clk)) { + dev_err(dev, "Failed to lookup clock base\n"); + return PTR_ERR(nfc->reg_clk); + } + + irq = platform_get_irq(pdev, 0); + if (irq < 0) { + dev_err(dev, "no NFC IRQ resource\n"); + return -EINVAL; + } + + ret = meson_nfc_clk_init(nfc); + if (ret) { + dev_err(dev, "failed to initialize NAND clock\n"); + return ret; + } + + writel(0, nfc->reg_base + NFC_REG_CFG); + ret = devm_request_irq(dev, irq, meson_nfc_irq, 0, dev_name(dev), nfc); + if (ret) { + dev_err(dev, "failed to request NFC IRQ\n"); + ret = -EINVAL; + goto err_clk; + } + + ret = dma_set_mask(dev, DMA_BIT_MASK(32)); + if (ret) { + dev_err(dev, "failed to set DMA mask\n"); + goto err_clk; + } + + platform_set_drvdata(pdev, nfc); + + ret = meson_nfc_nand_chips_init(dev, nfc); + if (ret) { + dev_err(dev, "failed to init NAND chips\n"); + goto err_clk; + } + + return 0; +err_clk: + meson_nfc_disable_clk(nfc); + return ret; +} + +static int meson_nfc_remove(struct platform_device *pdev) +{ + struct meson_nfc *nfc = platform_get_drvdata(pdev); + int ret; + + ret = meson_nfc_nand_chip_cleanup(nfc); + if (ret) + return ret; + + meson_nfc_disable_clk(nfc); + + platform_set_drvdata(pdev, NULL); + + return 0; +} + +static struct platform_driver meson_nfc_driver = { + .probe = meson_nfc_probe, + .remove = meson_nfc_remove, + .driver = { + .name = "meson-nand", + .of_match_table = meson_nfc_id_table, + }, +}; +module_platform_driver(meson_nfc_driver); + +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_AUTHOR("Liang Yang <liang.yang@amlogic.com>"); +MODULE_DESCRIPTION("Amlogic's Meson NAND Flash Controller driver"); diff --git a/drivers/mtd/nand/raw/mtk_ecc.c b/drivers/mtd/nand/raw/mtk_ecc.c index 6432bd70c3b3..05b0c19d72d9 100644 --- a/drivers/mtd/nand/raw/mtk_ecc.c +++ b/drivers/mtd/nand/raw/mtk_ecc.c @@ -267,11 +267,15 @@ static struct mtk_ecc *mtk_ecc_get(struct device_node *np) struct mtk_ecc *ecc; pdev = of_find_device_by_node(np); - if (!pdev || !platform_get_drvdata(pdev)) + if (!pdev) return ERR_PTR(-EPROBE_DEFER); - get_device(&pdev->dev); ecc = platform_get_drvdata(pdev); + if (!ecc) { + put_device(&pdev->dev); + return ERR_PTR(-EPROBE_DEFER); + } + clk_prepare_enable(ecc->clk); mtk_ecc_hw_init(ecc); diff --git a/drivers/mtd/nand/raw/mtk_nand.c b/drivers/mtd/nand/raw/mtk_nand.c index b6b4602f5132..2c0e09187773 100644 --- a/drivers/mtd/nand/raw/mtk_nand.c +++ b/drivers/mtd/nand/raw/mtk_nand.c @@ -1451,8 +1451,7 @@ static int mtk_nfc_probe(struct platform_device *pdev) if (!nfc) return -ENOMEM; - spin_lock_init(&nfc->controller.lock); - init_waitqueue_head(&nfc->controller.wq); + nand_controller_init(&nfc->controller); INIT_LIST_HEAD(&nfc->chips); nfc->controller.ops = &mtk_nfc_controller_ops; diff --git a/drivers/mtd/nand/raw/nand_base.c b/drivers/mtd/nand/raw/nand_base.c index 839494ac457c..ddd396e93e32 100644 --- a/drivers/mtd/nand/raw/nand_base.c +++ b/drivers/mtd/nand/raw/nand_base.c @@ -278,11 +278,8 @@ EXPORT_SYMBOL_GPL(nand_deselect_target); static void nand_release_device(struct nand_chip *chip) { /* Release the controller and the chip */ - spin_lock(&chip->controller->lock); - chip->controller->active = NULL; - chip->state = FL_READY; - wake_up(&chip->controller->wq); - spin_unlock(&chip->controller->lock); + mutex_unlock(&chip->controller->lock); + mutex_unlock(&chip->lock); } /** @@ -331,57 +328,23 @@ static int nand_isbad_bbm(struct nand_chip *chip, loff_t ofs) } /** - * panic_nand_get_device - [GENERIC] Get chip for selected access - * @chip: the nand chip descriptor - * @new_state: the state which is requested - * - * Used when in panic, no locks are taken. - */ -static void panic_nand_get_device(struct nand_chip *chip, int new_state) -{ - /* Hardware controller shared among independent devices */ - chip->controller->active = chip; - chip->state = new_state; -} - -/** * nand_get_device - [GENERIC] Get chip for selected access * @chip: NAND chip structure - * @new_state: the state which is requested * - * Get the device and lock it for exclusive access + * Lock the device and its controller for exclusive access + * + * Return: -EBUSY if the chip has been suspended, 0 otherwise */ -static int -nand_get_device(struct nand_chip *chip, int new_state) +static int nand_get_device(struct nand_chip *chip) { - spinlock_t *lock = &chip->controller->lock; - wait_queue_head_t *wq = &chip->controller->wq; - DECLARE_WAITQUEUE(wait, current); -retry: - spin_lock(lock); - - /* Hardware controller shared among independent devices */ - if (!chip->controller->active) - chip->controller->active = chip; - - if (chip->controller->active == chip && chip->state == FL_READY) { - chip->state = new_state; - spin_unlock(lock); - return 0; - } - if (new_state == FL_PM_SUSPENDED) { - if (chip->controller->active->state == FL_PM_SUSPENDED) { - chip->state = FL_PM_SUSPENDED; - spin_unlock(lock); - return 0; - } + mutex_lock(&chip->lock); + if (chip->suspended) { + mutex_unlock(&chip->lock); + return -EBUSY; } - set_current_state(TASK_UNINTERRUPTIBLE); - add_wait_queue(wq, &wait); - spin_unlock(lock); - schedule(); - remove_wait_queue(wq, &wait); - goto retry; + mutex_lock(&chip->controller->lock); + + return 0; } /** @@ -458,7 +421,7 @@ static int nand_do_write_oob(struct nand_chip *chip, loff_t to, struct mtd_oob_ops *ops) { struct mtd_info *mtd = nand_to_mtd(chip); - int chipnr, page, status, len; + int chipnr, page, status, len, ret; pr_debug("%s: to = 0x%08x, len = %i\n", __func__, (unsigned int)to, (int)ops->ooblen); @@ -480,7 +443,9 @@ static int nand_do_write_oob(struct nand_chip *chip, loff_t to, * if we don't do this. I have no clue why, but I seem to have 'fixed' * it in the doc2000 driver in August 1999. dwmw2. */ - nand_reset(chip, chipnr); + ret = nand_reset(chip, chipnr); + if (ret) + return ret; nand_select_target(chip, chipnr); @@ -603,7 +568,10 @@ static int nand_block_markbad_lowlevel(struct nand_chip *chip, loff_t ofs) nand_erase_nand(chip, &einfo, 0); /* Write bad block marker to OOB */ - nand_get_device(chip, FL_WRITING); + ret = nand_get_device(chip); + if (ret) + return ret; + ret = nand_markbad_bbm(chip, ofs); nand_release_device(chip); } @@ -3581,7 +3549,9 @@ static int nand_read_oob(struct mtd_info *mtd, loff_t from, ops->mode != MTD_OPS_RAW) return -ENOTSUPP; - nand_get_device(chip, FL_READING); + ret = nand_get_device(chip); + if (ret) + return ret; if (!ops->datbuf) ret = nand_do_read_oob(chip, from, ops); @@ -4100,9 +4070,6 @@ static int panic_nand_write(struct mtd_info *mtd, loff_t to, size_t len, struct mtd_oob_ops ops; int ret; - /* Grab the device */ - panic_nand_get_device(chip, FL_WRITING); - nand_select_target(chip, chipnr); /* Wait for the device to get ready */ @@ -4133,7 +4100,9 @@ static int nand_write_oob(struct mtd_info *mtd, loff_t to, ops->retlen = 0; - nand_get_device(chip, FL_WRITING); + ret = nand_get_device(chip); + if (ret) + return ret; switch (ops->mode) { case MTD_OPS_PLACE_OOB: @@ -4156,23 +4125,6 @@ out: } /** - * single_erase - [GENERIC] NAND standard block erase command function - * @chip: NAND chip object - * @page: the page address of the block which will be erased - * - * Standard erase command for NAND chips. Returns NAND status. - */ -static int single_erase(struct nand_chip *chip, int page) -{ - unsigned int eraseblock; - - /* Send commands to erase a block */ - eraseblock = page >> (chip->phys_erase_shift - chip->page_shift); - - return nand_erase_op(chip, eraseblock); -} - -/** * nand_erase - [MTD Interface] erase block(s) * @mtd: MTD device structure * @instr: erase instruction @@ -4195,7 +4147,7 @@ static int nand_erase(struct mtd_info *mtd, struct erase_info *instr) int nand_erase_nand(struct nand_chip *chip, struct erase_info *instr, int allowbbt) { - int page, status, pages_per_block, ret, chipnr; + int page, pages_per_block, ret, chipnr; loff_t len; pr_debug("%s: start = 0x%012llx, len = %llu\n", @@ -4206,7 +4158,9 @@ int nand_erase_nand(struct nand_chip *chip, struct erase_info *instr, return -EINVAL; /* Grab the lock and see if the device is available */ - nand_get_device(chip, FL_ERASING); + ret = nand_get_device(chip); + if (ret) + return ret; /* Shift to get first page */ page = (int)(instr->addr >> chip->page_shift); @@ -4247,17 +4201,11 @@ int nand_erase_nand(struct nand_chip *chip, struct erase_info *instr, (page + pages_per_block)) chip->pagebuf = -1; - if (chip->legacy.erase) - status = chip->legacy.erase(chip, - page & chip->pagemask); - else - status = single_erase(chip, page & chip->pagemask); - - /* See if block erase succeeded */ - if (status) { + ret = nand_erase_op(chip, (page & chip->pagemask) >> + (chip->phys_erase_shift - chip->page_shift)); + if (ret) { pr_debug("%s: failed erase, page 0x%08x\n", __func__, page); - ret = -EIO; instr->fail_addr = ((loff_t)page << chip->page_shift); goto erase_exit; @@ -4299,7 +4247,7 @@ static void nand_sync(struct mtd_info *mtd) pr_debug("%s: called\n", __func__); /* Grab the lock and see if the device is available */ - nand_get_device(chip, FL_SYNCING); + WARN_ON(nand_get_device(chip)); /* Release it and go back */ nand_release_device(chip); } @@ -4316,7 +4264,10 @@ static int nand_block_isbad(struct mtd_info *mtd, loff_t offs) int ret; /* Select the NAND device */ - nand_get_device(chip, FL_READING); + ret = nand_get_device(chip); + if (ret) + return ret; + nand_select_target(chip, chipnr); ret = nand_block_checkbad(chip, offs, 0); @@ -4389,7 +4340,13 @@ static int nand_max_bad_blocks(struct mtd_info *mtd, loff_t ofs, size_t len) */ static int nand_suspend(struct mtd_info *mtd) { - return nand_get_device(mtd_to_nand(mtd), FL_PM_SUSPENDED); + struct nand_chip *chip = mtd_to_nand(mtd); + + mutex_lock(&chip->lock); + chip->suspended = 1; + mutex_unlock(&chip->lock); + + return 0; } /** @@ -4400,11 +4357,13 @@ static void nand_resume(struct mtd_info *mtd) { struct nand_chip *chip = mtd_to_nand(mtd); - if (chip->state == FL_PM_SUSPENDED) - nand_release_device(chip); + mutex_lock(&chip->lock); + if (chip->suspended) + chip->suspended = 0; else pr_err("%s called for a chip which is not in suspended state\n", __func__); + mutex_unlock(&chip->lock); } /** @@ -4414,7 +4373,7 @@ static void nand_resume(struct mtd_info *mtd) */ static void nand_shutdown(struct mtd_info *mtd) { - nand_get_device(mtd_to_nand(mtd), FL_PM_SUSPENDED); + nand_suspend(mtd); } /* Set default functions */ @@ -5019,6 +4978,8 @@ static int nand_scan_ident(struct nand_chip *chip, unsigned int maxchips, /* Assume all dies are deselected when we enter nand_scan_ident(). */ chip->cur_cs = -1; + mutex_init(&chip->lock); + /* Enforce the right timings for reset/detection */ onfi_fill_data_interface(chip, NAND_SDR_IFACE, 0); @@ -5061,11 +5022,15 @@ static int nand_scan_ident(struct nand_chip *chip, unsigned int maxchips, u8 id[2]; /* See comment in nand_get_flash_type for reset */ - nand_reset(chip, i); + ret = nand_reset(chip, i); + if (ret) + break; nand_select_target(chip, i); /* Send the command for reading device ID */ - nand_readid_op(chip, 0, id, sizeof(id)); + ret = nand_readid_op(chip, 0, id, sizeof(id)); + if (ret) + break; /* Read manufacturer and device IDs */ if (nand_maf_id != id[0] || nand_dev_id != id[1]) { nand_deselect_target(chip); @@ -5556,6 +5521,7 @@ static int nand_scan_tail(struct nand_chip *chip) } if (!ecc->read_page) ecc->read_page = nand_read_page_hwecc_oob_first; + /* fall through */ case NAND_ECC_HW: /* Use standard hwecc read page function? */ @@ -5575,6 +5541,7 @@ static int nand_scan_tail(struct nand_chip *chip) ecc->read_subpage = nand_read_subpage; if (!ecc->write_subpage && ecc->hwctl && ecc->calculate) ecc->write_subpage = nand_write_subpage_hwecc; + /* fall through */ case NAND_ECC_HW_SYNDROME: if ((!ecc->calculate || !ecc->correct || !ecc->hwctl) && @@ -5612,6 +5579,7 @@ static int nand_scan_tail(struct nand_chip *chip) ecc->size, mtd->writesize); ecc->mode = NAND_ECC_SOFT; ecc->algo = NAND_ECC_HAMMING; + /* fall through */ case NAND_ECC_SOFT: ret = nand_set_ecc_soft_ops(chip); @@ -5718,9 +5686,6 @@ static int nand_scan_tail(struct nand_chip *chip) } chip->subpagesize = mtd->writesize >> mtd->subpage_sft; - /* Initialize state */ - chip->state = FL_READY; - /* Invalidate the pagebuffer reference */ chip->pagebuf = -1; diff --git a/drivers/mtd/nand/raw/nand_legacy.c b/drivers/mtd/nand/raw/nand_legacy.c index 43575943f13b..f2526ec616a6 100644 --- a/drivers/mtd/nand/raw/nand_legacy.c +++ b/drivers/mtd/nand/raw/nand_legacy.c @@ -331,6 +331,7 @@ static void nand_command(struct nand_chip *chip, unsigned int command, */ if (column == -1 && page_addr == -1) return; + /* fall through */ default: /* @@ -483,7 +484,7 @@ static void nand_command_lp(struct nand_chip *chip, unsigned int command, chip->legacy.cmd_ctrl(chip, NAND_CMD_NONE, NAND_NCE | NAND_CTRL_CHANGE); - /* This applies to read commands */ + /* fall through - This applies to read commands */ default: /* * If we don't have access to the busy pin, we apply the given diff --git a/drivers/mtd/nand/raw/omap2.c b/drivers/mtd/nand/raw/omap2.c index 68e8b9f7f372..8f280a2962c8 100644 --- a/drivers/mtd/nand/raw/omap2.c +++ b/drivers/mtd/nand/raw/omap2.c @@ -994,12 +994,9 @@ static int omap_wait(struct nand_chip *this) { struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(this)); unsigned long timeo = jiffies; - int status, state = this->state; + int status; - if (state == FL_ERASING) - timeo += msecs_to_jiffies(400); - else - timeo += msecs_to_jiffies(20); + timeo += msecs_to_jiffies(400); writeb(NAND_CMD_STATUS & 0xFF, info->reg.gpmc_nand_command); while (time_before(jiffies, timeo)) { @@ -2173,11 +2170,8 @@ static const struct nand_controller_ops omap_nand_controller_ops = { }; /* Shared among all NAND instances to synchronize access to the ECC Engine */ -static struct nand_controller omap_gpmc_controller = { - .lock = __SPIN_LOCK_UNLOCKED(omap_gpmc_controller.lock), - .wq = __WAIT_QUEUE_HEAD_INITIALIZER(omap_gpmc_controller.wq), - .ops = &omap_nand_controller_ops, -}; +static struct nand_controller omap_gpmc_controller; +static bool omap_gpmc_controller_initialized; static int omap_nand_probe(struct platform_device *pdev) { @@ -2227,6 +2221,12 @@ static int omap_nand_probe(struct platform_device *pdev) info->phys_base = res->start; + if (!omap_gpmc_controller_initialized) { + omap_gpmc_controller.ops = &omap_nand_controller_ops; + nand_controller_init(&omap_gpmc_controller); + omap_gpmc_controller_initialized = true; + } + nand_chip->controller = &omap_gpmc_controller; nand_chip->legacy.IO_ADDR_W = nand_chip->legacy.IO_ADDR_R; diff --git a/drivers/mtd/nand/raw/r852.c b/drivers/mtd/nand/raw/r852.c index c01422d953dd..86456216fb93 100644 --- a/drivers/mtd/nand/raw/r852.c +++ b/drivers/mtd/nand/raw/r852.c @@ -369,8 +369,7 @@ static int r852_wait(struct nand_chip *chip) unsigned long timeout; u8 status; - timeout = jiffies + (chip->state == FL_ERASING ? - msecs_to_jiffies(400) : msecs_to_jiffies(20)); + timeout = jiffies + msecs_to_jiffies(400); while (time_before(jiffies, timeout)) if (chip->legacy.dev_ready(chip)) diff --git a/drivers/mtd/nand/raw/stm32_fmc2_nand.c b/drivers/mtd/nand/raw/stm32_fmc2_nand.c new file mode 100644 index 000000000000..999ca6a66036 --- /dev/null +++ b/drivers/mtd/nand/raw/stm32_fmc2_nand.c @@ -0,0 +1,2073 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) STMicroelectronics 2018 + * Author: Christophe Kerello <christophe.kerello@st.com> + */ + +#include <linux/clk.h> +#include <linux/dmaengine.h> +#include <linux/dma-mapping.h> +#include <linux/errno.h> +#include <linux/interrupt.h> +#include <linux/iopoll.h> +#include <linux/module.h> +#include <linux/mtd/rawnand.h> +#include <linux/pinctrl/consumer.h> +#include <linux/platform_device.h> +#include <linux/reset.h> + +/* Bad block marker length */ +#define FMC2_BBM_LEN 2 + +/* ECC step size */ +#define FMC2_ECC_STEP_SIZE 512 + +/* BCHDSRx registers length */ +#define FMC2_BCHDSRS_LEN 20 + +/* HECCR length */ +#define FMC2_HECCR_LEN 4 + +/* Max requests done for a 8k nand page size */ +#define FMC2_MAX_SG 16 + +/* Max chip enable */ +#define FMC2_MAX_CE 2 + +/* Max ECC buffer length */ +#define FMC2_MAX_ECC_BUF_LEN (FMC2_BCHDSRS_LEN * FMC2_MAX_SG) + +/* Timings */ +#define FMC2_THIZ 1 +#define FMC2_TIO 8000 +#define FMC2_TSYNC 3000 +#define FMC2_PCR_TIMING_MASK 0xf +#define FMC2_PMEM_PATT_TIMING_MASK 0xff + +/* FMC2 Controller Registers */ +#define FMC2_BCR1 0x0 +#define FMC2_PCR 0x80 +#define FMC2_SR 0x84 +#define FMC2_PMEM 0x88 +#define FMC2_PATT 0x8c +#define FMC2_HECCR 0x94 +#define FMC2_CSQCR 0x200 +#define FMC2_CSQCFGR1 0x204 +#define FMC2_CSQCFGR2 0x208 +#define FMC2_CSQCFGR3 0x20c +#define FMC2_CSQAR1 0x210 +#define FMC2_CSQAR2 0x214 +#define FMC2_CSQIER 0x220 +#define FMC2_CSQISR 0x224 +#define FMC2_CSQICR 0x228 +#define FMC2_CSQEMSR 0x230 +#define FMC2_BCHIER 0x250 +#define FMC2_BCHISR 0x254 +#define FMC2_BCHICR 0x258 +#define FMC2_BCHPBR1 0x260 +#define FMC2_BCHPBR2 0x264 +#define FMC2_BCHPBR3 0x268 +#define FMC2_BCHPBR4 0x26c +#define FMC2_BCHDSR0 0x27c +#define FMC2_BCHDSR1 0x280 +#define FMC2_BCHDSR2 0x284 +#define FMC2_BCHDSR3 0x288 +#define FMC2_BCHDSR4 0x28c + +/* Register: FMC2_BCR1 */ +#define FMC2_BCR1_FMC2EN BIT(31) + +/* Register: FMC2_PCR */ +#define FMC2_PCR_PWAITEN BIT(1) +#define FMC2_PCR_PBKEN BIT(2) +#define FMC2_PCR_PWID_MASK GENMASK(5, 4) +#define FMC2_PCR_PWID(x) (((x) & 0x3) << 4) +#define FMC2_PCR_PWID_BUSWIDTH_8 0 +#define FMC2_PCR_PWID_BUSWIDTH_16 1 +#define FMC2_PCR_ECCEN BIT(6) +#define FMC2_PCR_ECCALG BIT(8) +#define FMC2_PCR_TCLR_MASK GENMASK(12, 9) +#define FMC2_PCR_TCLR(x) (((x) & 0xf) << 9) +#define FMC2_PCR_TCLR_DEFAULT 0xf +#define FMC2_PCR_TAR_MASK GENMASK(16, 13) +#define FMC2_PCR_TAR(x) (((x) & 0xf) << 13) +#define FMC2_PCR_TAR_DEFAULT 0xf +#define FMC2_PCR_ECCSS_MASK GENMASK(19, 17) +#define FMC2_PCR_ECCSS(x) (((x) & 0x7) << 17) +#define FMC2_PCR_ECCSS_512 1 +#define FMC2_PCR_ECCSS_2048 3 +#define FMC2_PCR_BCHECC BIT(24) +#define FMC2_PCR_WEN BIT(25) + +/* Register: FMC2_SR */ +#define FMC2_SR_NWRF BIT(6) + +/* Register: FMC2_PMEM */ +#define FMC2_PMEM_MEMSET(x) (((x) & 0xff) << 0) +#define FMC2_PMEM_MEMWAIT(x) (((x) & 0xff) << 8) +#define FMC2_PMEM_MEMHOLD(x) (((x) & 0xff) << 16) +#define FMC2_PMEM_MEMHIZ(x) (((x) & 0xff) << 24) +#define FMC2_PMEM_DEFAULT 0x0a0a0a0a + +/* Register: FMC2_PATT */ +#define FMC2_PATT_ATTSET(x) (((x) & 0xff) << 0) +#define FMC2_PATT_ATTWAIT(x) (((x) & 0xff) << 8) +#define FMC2_PATT_ATTHOLD(x) (((x) & 0xff) << 16) +#define FMC2_PATT_ATTHIZ(x) (((x) & 0xff) << 24) +#define FMC2_PATT_DEFAULT 0x0a0a0a0a + +/* Register: FMC2_CSQCR */ +#define FMC2_CSQCR_CSQSTART BIT(0) + +/* Register: FMC2_CSQCFGR1 */ +#define FMC2_CSQCFGR1_CMD2EN BIT(1) +#define FMC2_CSQCFGR1_DMADEN BIT(2) +#define FMC2_CSQCFGR1_ACYNBR(x) (((x) & 0x7) << 4) +#define FMC2_CSQCFGR1_CMD1(x) (((x) & 0xff) << 8) +#define FMC2_CSQCFGR1_CMD2(x) (((x) & 0xff) << 16) +#define FMC2_CSQCFGR1_CMD1T BIT(24) +#define FMC2_CSQCFGR1_CMD2T BIT(25) + +/* Register: FMC2_CSQCFGR2 */ +#define FMC2_CSQCFGR2_SQSDTEN BIT(0) +#define FMC2_CSQCFGR2_RCMD2EN BIT(1) +#define FMC2_CSQCFGR2_DMASEN BIT(2) +#define FMC2_CSQCFGR2_RCMD1(x) (((x) & 0xff) << 8) +#define FMC2_CSQCFGR2_RCMD2(x) (((x) & 0xff) << 16) +#define FMC2_CSQCFGR2_RCMD1T BIT(24) +#define FMC2_CSQCFGR2_RCMD2T BIT(25) + +/* Register: FMC2_CSQCFGR3 */ +#define FMC2_CSQCFGR3_SNBR(x) (((x) & 0x1f) << 8) +#define FMC2_CSQCFGR3_AC1T BIT(16) +#define FMC2_CSQCFGR3_AC2T BIT(17) +#define FMC2_CSQCFGR3_AC3T BIT(18) +#define FMC2_CSQCFGR3_AC4T BIT(19) +#define FMC2_CSQCFGR3_AC5T BIT(20) +#define FMC2_CSQCFGR3_SDT BIT(21) +#define FMC2_CSQCFGR3_RAC1T BIT(22) +#define FMC2_CSQCFGR3_RAC2T BIT(23) + +/* Register: FMC2_CSQCAR1 */ +#define FMC2_CSQCAR1_ADDC1(x) (((x) & 0xff) << 0) +#define FMC2_CSQCAR1_ADDC2(x) (((x) & 0xff) << 8) +#define FMC2_CSQCAR1_ADDC3(x) (((x) & 0xff) << 16) +#define FMC2_CSQCAR1_ADDC4(x) (((x) & 0xff) << 24) + +/* Register: FMC2_CSQCAR2 */ +#define FMC2_CSQCAR2_ADDC5(x) (((x) & 0xff) << 0) +#define FMC2_CSQCAR2_NANDCEN(x) (((x) & 0x3) << 10) +#define FMC2_CSQCAR2_SAO(x) (((x) & 0xffff) << 16) + +/* Register: FMC2_CSQIER */ +#define FMC2_CSQIER_TCIE BIT(0) + +/* Register: FMC2_CSQICR */ +#define FMC2_CSQICR_CLEAR_IRQ GENMASK(4, 0) + +/* Register: FMC2_CSQEMSR */ +#define FMC2_CSQEMSR_SEM GENMASK(15, 0) + +/* Register: FMC2_BCHIER */ +#define FMC2_BCHIER_DERIE BIT(1) +#define FMC2_BCHIER_EPBRIE BIT(4) + +/* Register: FMC2_BCHICR */ +#define FMC2_BCHICR_CLEAR_IRQ GENMASK(4, 0) + +/* Register: FMC2_BCHDSR0 */ +#define FMC2_BCHDSR0_DUE BIT(0) +#define FMC2_BCHDSR0_DEF BIT(1) +#define FMC2_BCHDSR0_DEN_MASK GENMASK(7, 4) +#define FMC2_BCHDSR0_DEN_SHIFT 4 + +/* Register: FMC2_BCHDSR1 */ +#define FMC2_BCHDSR1_EBP1_MASK GENMASK(12, 0) +#define FMC2_BCHDSR1_EBP2_MASK GENMASK(28, 16) +#define FMC2_BCHDSR1_EBP2_SHIFT 16 + +/* Register: FMC2_BCHDSR2 */ +#define FMC2_BCHDSR2_EBP3_MASK GENMASK(12, 0) +#define FMC2_BCHDSR2_EBP4_MASK GENMASK(28, 16) +#define FMC2_BCHDSR2_EBP4_SHIFT 16 + +/* Register: FMC2_BCHDSR3 */ +#define FMC2_BCHDSR3_EBP5_MASK GENMASK(12, 0) +#define FMC2_BCHDSR3_EBP6_MASK GENMASK(28, 16) +#define FMC2_BCHDSR3_EBP6_SHIFT 16 + +/* Register: FMC2_BCHDSR4 */ +#define FMC2_BCHDSR4_EBP7_MASK GENMASK(12, 0) +#define FMC2_BCHDSR4_EBP8_MASK GENMASK(28, 16) +#define FMC2_BCHDSR4_EBP8_SHIFT 16 + +enum stm32_fmc2_ecc { + FMC2_ECC_HAM = 1, + FMC2_ECC_BCH4 = 4, + FMC2_ECC_BCH8 = 8 +}; + +enum stm32_fmc2_irq_state { + FMC2_IRQ_UNKNOWN = 0, + FMC2_IRQ_BCH, + FMC2_IRQ_SEQ +}; + +struct stm32_fmc2_timings { + u8 tclr; + u8 tar; + u8 thiz; + u8 twait; + u8 thold_mem; + u8 tset_mem; + u8 thold_att; + u8 tset_att; +}; + +struct stm32_fmc2_nand { + struct nand_chip chip; + struct stm32_fmc2_timings timings; + int ncs; + int cs_used[FMC2_MAX_CE]; +}; + +static inline struct stm32_fmc2_nand *to_fmc2_nand(struct nand_chip *chip) +{ + return container_of(chip, struct stm32_fmc2_nand, chip); +} + +struct stm32_fmc2_nfc { + struct nand_controller base; + struct stm32_fmc2_nand nand; + struct device *dev; + void __iomem *io_base; + void __iomem *data_base[FMC2_MAX_CE]; + void __iomem *cmd_base[FMC2_MAX_CE]; + void __iomem *addr_base[FMC2_MAX_CE]; + phys_addr_t io_phys_addr; + phys_addr_t data_phys_addr[FMC2_MAX_CE]; + struct clk *clk; + u8 irq_state; + + struct dma_chan *dma_tx_ch; + struct dma_chan *dma_rx_ch; + struct dma_chan *dma_ecc_ch; + struct sg_table dma_data_sg; + struct sg_table dma_ecc_sg; + u8 *ecc_buf; + int dma_ecc_len; + + struct completion complete; + struct completion dma_data_complete; + struct completion dma_ecc_complete; + + u8 cs_assigned; + int cs_sel; +}; + +static inline struct stm32_fmc2_nfc *to_stm32_nfc(struct nand_controller *base) +{ + return container_of(base, struct stm32_fmc2_nfc, base); +} + +/* Timings configuration */ +static void stm32_fmc2_timings_init(struct nand_chip *chip) +{ + struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller); + struct stm32_fmc2_nand *nand = to_fmc2_nand(chip); + struct stm32_fmc2_timings *timings = &nand->timings; + u32 pcr = readl_relaxed(fmc2->io_base + FMC2_PCR); + u32 pmem, patt; + + /* Set tclr/tar timings */ + pcr &= ~FMC2_PCR_TCLR_MASK; + pcr |= FMC2_PCR_TCLR(timings->tclr); + pcr &= ~FMC2_PCR_TAR_MASK; + pcr |= FMC2_PCR_TAR(timings->tar); + + /* Set tset/twait/thold/thiz timings in common bank */ + pmem = FMC2_PMEM_MEMSET(timings->tset_mem); + pmem |= FMC2_PMEM_MEMWAIT(timings->twait); + pmem |= FMC2_PMEM_MEMHOLD(timings->thold_mem); + pmem |= FMC2_PMEM_MEMHIZ(timings->thiz); + + /* Set tset/twait/thold/thiz timings in attribut bank */ + patt = FMC2_PATT_ATTSET(timings->tset_att); + patt |= FMC2_PATT_ATTWAIT(timings->twait); + patt |= FMC2_PATT_ATTHOLD(timings->thold_att); + patt |= FMC2_PATT_ATTHIZ(timings->thiz); + + writel_relaxed(pcr, fmc2->io_base + FMC2_PCR); + writel_relaxed(pmem, fmc2->io_base + FMC2_PMEM); + writel_relaxed(patt, fmc2->io_base + FMC2_PATT); +} + +/* Controller configuration */ +static void stm32_fmc2_setup(struct nand_chip *chip) +{ + struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller); + u32 pcr = readl_relaxed(fmc2->io_base + FMC2_PCR); + + /* Configure ECC algorithm (default configuration is Hamming) */ + pcr &= ~FMC2_PCR_ECCALG; + pcr &= ~FMC2_PCR_BCHECC; + if (chip->ecc.strength == FMC2_ECC_BCH8) { + pcr |= FMC2_PCR_ECCALG; + pcr |= FMC2_PCR_BCHECC; + } else if (chip->ecc.strength == FMC2_ECC_BCH4) { + pcr |= FMC2_PCR_ECCALG; + } + + /* Set buswidth */ + pcr &= ~FMC2_PCR_PWID_MASK; + if (chip->options & NAND_BUSWIDTH_16) + pcr |= FMC2_PCR_PWID(FMC2_PCR_PWID_BUSWIDTH_16); + + /* Set ECC sector size */ + pcr &= ~FMC2_PCR_ECCSS_MASK; + pcr |= FMC2_PCR_ECCSS(FMC2_PCR_ECCSS_512); + + writel_relaxed(pcr, fmc2->io_base + FMC2_PCR); +} + +/* Select target */ +static int stm32_fmc2_select_chip(struct nand_chip *chip, int chipnr) +{ + struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller); + struct stm32_fmc2_nand *nand = to_fmc2_nand(chip); + struct dma_slave_config dma_cfg; + int ret; + + if (nand->cs_used[chipnr] == fmc2->cs_sel) + return 0; + + fmc2->cs_sel = nand->cs_used[chipnr]; + + /* FMC2 setup routine */ + stm32_fmc2_setup(chip); + + /* Apply timings */ + stm32_fmc2_timings_init(chip); + + if (fmc2->dma_tx_ch && fmc2->dma_rx_ch) { + memset(&dma_cfg, 0, sizeof(dma_cfg)); + dma_cfg.src_addr = fmc2->data_phys_addr[fmc2->cs_sel]; + dma_cfg.dst_addr = fmc2->data_phys_addr[fmc2->cs_sel]; + dma_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + dma_cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + dma_cfg.src_maxburst = 32; + dma_cfg.dst_maxburst = 32; + + ret = dmaengine_slave_config(fmc2->dma_tx_ch, &dma_cfg); + if (ret) { + dev_err(fmc2->dev, "tx DMA engine slave config failed\n"); + return ret; + } + + ret = dmaengine_slave_config(fmc2->dma_rx_ch, &dma_cfg); + if (ret) { + dev_err(fmc2->dev, "rx DMA engine slave config failed\n"); + return ret; + } + } + + if (fmc2->dma_ecc_ch) { + /* + * Hamming: we read HECCR register + * BCH4/BCH8: we read BCHDSRSx registers + */ + memset(&dma_cfg, 0, sizeof(dma_cfg)); + dma_cfg.src_addr = fmc2->io_phys_addr; + dma_cfg.src_addr += chip->ecc.strength == FMC2_ECC_HAM ? + FMC2_HECCR : FMC2_BCHDSR0; + dma_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + + ret = dmaengine_slave_config(fmc2->dma_ecc_ch, &dma_cfg); + if (ret) { + dev_err(fmc2->dev, "ECC DMA engine slave config failed\n"); + return ret; + } + + /* Calculate ECC length needed for one sector */ + fmc2->dma_ecc_len = chip->ecc.strength == FMC2_ECC_HAM ? + FMC2_HECCR_LEN : FMC2_BCHDSRS_LEN; + } + + return 0; +} + +/* Set bus width to 16-bit or 8-bit */ +static void stm32_fmc2_set_buswidth_16(struct stm32_fmc2_nfc *fmc2, bool set) +{ + u32 pcr = readl_relaxed(fmc2->io_base + FMC2_PCR); + + pcr &= ~FMC2_PCR_PWID_MASK; + if (set) + pcr |= FMC2_PCR_PWID(FMC2_PCR_PWID_BUSWIDTH_16); + writel_relaxed(pcr, fmc2->io_base + FMC2_PCR); +} + +/* Enable/disable ECC */ +static void stm32_fmc2_set_ecc(struct stm32_fmc2_nfc *fmc2, bool enable) +{ + u32 pcr = readl(fmc2->io_base + FMC2_PCR); + + pcr &= ~FMC2_PCR_ECCEN; + if (enable) + pcr |= FMC2_PCR_ECCEN; + writel(pcr, fmc2->io_base + FMC2_PCR); +} + +/* Enable irq sources in case of the sequencer is used */ +static inline void stm32_fmc2_enable_seq_irq(struct stm32_fmc2_nfc *fmc2) +{ + u32 csqier = readl_relaxed(fmc2->io_base + FMC2_CSQIER); + + csqier |= FMC2_CSQIER_TCIE; + + fmc2->irq_state = FMC2_IRQ_SEQ; + + writel_relaxed(csqier, fmc2->io_base + FMC2_CSQIER); +} + +/* Disable irq sources in case of the sequencer is used */ +static inline void stm32_fmc2_disable_seq_irq(struct stm32_fmc2_nfc *fmc2) +{ + u32 csqier = readl_relaxed(fmc2->io_base + FMC2_CSQIER); + + csqier &= ~FMC2_CSQIER_TCIE; + + writel_relaxed(csqier, fmc2->io_base + FMC2_CSQIER); + + fmc2->irq_state = FMC2_IRQ_UNKNOWN; +} + +/* Clear irq sources in case of the sequencer is used */ +static inline void stm32_fmc2_clear_seq_irq(struct stm32_fmc2_nfc *fmc2) +{ + writel_relaxed(FMC2_CSQICR_CLEAR_IRQ, fmc2->io_base + FMC2_CSQICR); +} + +/* Enable irq sources in case of bch is used */ +static inline void stm32_fmc2_enable_bch_irq(struct stm32_fmc2_nfc *fmc2, + int mode) +{ + u32 bchier = readl_relaxed(fmc2->io_base + FMC2_BCHIER); + + if (mode == NAND_ECC_WRITE) + bchier |= FMC2_BCHIER_EPBRIE; + else + bchier |= FMC2_BCHIER_DERIE; + + fmc2->irq_state = FMC2_IRQ_BCH; + + writel_relaxed(bchier, fmc2->io_base + FMC2_BCHIER); +} + +/* Disable irq sources in case of bch is used */ +static inline void stm32_fmc2_disable_bch_irq(struct stm32_fmc2_nfc *fmc2) +{ + u32 bchier = readl_relaxed(fmc2->io_base + FMC2_BCHIER); + + bchier &= ~FMC2_BCHIER_DERIE; + bchier &= ~FMC2_BCHIER_EPBRIE; + + writel_relaxed(bchier, fmc2->io_base + FMC2_BCHIER); + + fmc2->irq_state = FMC2_IRQ_UNKNOWN; +} + +/* Clear irq sources in case of bch is used */ +static inline void stm32_fmc2_clear_bch_irq(struct stm32_fmc2_nfc *fmc2) +{ + writel_relaxed(FMC2_BCHICR_CLEAR_IRQ, fmc2->io_base + FMC2_BCHICR); +} + +/* + * Enable ECC logic and reset syndrome/parity bits previously calculated + * Syndrome/parity bits is cleared by setting the ECCEN bit to 0 + */ +static void stm32_fmc2_hwctl(struct nand_chip *chip, int mode) +{ + struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller); + + stm32_fmc2_set_ecc(fmc2, false); + + if (chip->ecc.strength != FMC2_ECC_HAM) { + u32 pcr = readl_relaxed(fmc2->io_base + FMC2_PCR); + + if (mode == NAND_ECC_WRITE) + pcr |= FMC2_PCR_WEN; + else + pcr &= ~FMC2_PCR_WEN; + writel_relaxed(pcr, fmc2->io_base + FMC2_PCR); + + reinit_completion(&fmc2->complete); + stm32_fmc2_clear_bch_irq(fmc2); + stm32_fmc2_enable_bch_irq(fmc2, mode); + } + + stm32_fmc2_set_ecc(fmc2, true); +} + +/* + * ECC Hamming calculation + * ECC is 3 bytes for 512 bytes of data (supports error correction up to + * max of 1-bit) + */ +static inline void stm32_fmc2_ham_set_ecc(const u32 ecc_sta, u8 *ecc) +{ + ecc[0] = ecc_sta; + ecc[1] = ecc_sta >> 8; + ecc[2] = ecc_sta >> 16; +} + +static int stm32_fmc2_ham_calculate(struct nand_chip *chip, const u8 *data, + u8 *ecc) +{ + struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller); + u32 sr, heccr; + int ret; + + ret = readl_relaxed_poll_timeout(fmc2->io_base + FMC2_SR, + sr, sr & FMC2_SR_NWRF, 10, 1000); + if (ret) { + dev_err(fmc2->dev, "ham timeout\n"); + return ret; + } + + heccr = readl_relaxed(fmc2->io_base + FMC2_HECCR); + + stm32_fmc2_ham_set_ecc(heccr, ecc); + + /* Disable ECC */ + stm32_fmc2_set_ecc(fmc2, false); + + return 0; +} + +static int stm32_fmc2_ham_correct(struct nand_chip *chip, u8 *dat, + u8 *read_ecc, u8 *calc_ecc) +{ + u8 bit_position = 0, b0, b1, b2; + u32 byte_addr = 0, b; + u32 i, shifting = 1; + + /* Indicate which bit and byte is faulty (if any) */ + b0 = read_ecc[0] ^ calc_ecc[0]; + b1 = read_ecc[1] ^ calc_ecc[1]; + b2 = read_ecc[2] ^ calc_ecc[2]; + b = b0 | (b1 << 8) | (b2 << 16); + + /* No errors */ + if (likely(!b)) + return 0; + + /* Calculate bit position */ + for (i = 0; i < 3; i++) { + switch (b % 4) { + case 2: + bit_position += shifting; + case 1: + break; + default: + return -EBADMSG; + } + shifting <<= 1; + b >>= 2; + } + + /* Calculate byte position */ + shifting = 1; + for (i = 0; i < 9; i++) { + switch (b % 4) { + case 2: + byte_addr += shifting; + case 1: + break; + default: + return -EBADMSG; + } + shifting <<= 1; + b >>= 2; + } + + /* Flip the bit */ + dat[byte_addr] ^= (1 << bit_position); + + return 1; +} + +/* + * ECC BCH calculation and correction + * ECC is 7/13 bytes for 512 bytes of data (supports error correction up to + * max of 4-bit/8-bit) + */ +static int stm32_fmc2_bch_calculate(struct nand_chip *chip, const u8 *data, + u8 *ecc) +{ + struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller); + u32 bchpbr; + + /* Wait until the BCH code is ready */ + if (!wait_for_completion_timeout(&fmc2->complete, + msecs_to_jiffies(1000))) { + dev_err(fmc2->dev, "bch timeout\n"); + stm32_fmc2_disable_bch_irq(fmc2); + return -ETIMEDOUT; + } + + /* Read parity bits */ + bchpbr = readl_relaxed(fmc2->io_base + FMC2_BCHPBR1); + ecc[0] = bchpbr; + ecc[1] = bchpbr >> 8; + ecc[2] = bchpbr >> 16; + ecc[3] = bchpbr >> 24; + + bchpbr = readl_relaxed(fmc2->io_base + FMC2_BCHPBR2); + ecc[4] = bchpbr; + ecc[5] = bchpbr >> 8; + ecc[6] = bchpbr >> 16; + + if (chip->ecc.strength == FMC2_ECC_BCH8) { + ecc[7] = bchpbr >> 24; + + bchpbr = readl_relaxed(fmc2->io_base + FMC2_BCHPBR3); + ecc[8] = bchpbr; + ecc[9] = bchpbr >> 8; + ecc[10] = bchpbr >> 16; + ecc[11] = bchpbr >> 24; + + bchpbr = readl_relaxed(fmc2->io_base + FMC2_BCHPBR4); + ecc[12] = bchpbr; + } + + /* Disable ECC */ + stm32_fmc2_set_ecc(fmc2, false); + + return 0; +} + +/* BCH algorithm correction */ +static int stm32_fmc2_bch_decode(int eccsize, u8 *dat, u32 *ecc_sta) +{ + u32 bchdsr0 = ecc_sta[0]; + u32 bchdsr1 = ecc_sta[1]; + u32 bchdsr2 = ecc_sta[2]; + u32 bchdsr3 = ecc_sta[3]; + u32 bchdsr4 = ecc_sta[4]; + u16 pos[8]; + int i, den; + unsigned int nb_errs = 0; + + /* No errors found */ + if (likely(!(bchdsr0 & FMC2_BCHDSR0_DEF))) + return 0; + + /* Too many errors detected */ + if (unlikely(bchdsr0 & FMC2_BCHDSR0_DUE)) + return -EBADMSG; + + pos[0] = bchdsr1 & FMC2_BCHDSR1_EBP1_MASK; + pos[1] = (bchdsr1 & FMC2_BCHDSR1_EBP2_MASK) >> FMC2_BCHDSR1_EBP2_SHIFT; + pos[2] = bchdsr2 & FMC2_BCHDSR2_EBP3_MASK; + pos[3] = (bchdsr2 & FMC2_BCHDSR2_EBP4_MASK) >> FMC2_BCHDSR2_EBP4_SHIFT; + pos[4] = bchdsr3 & FMC2_BCHDSR3_EBP5_MASK; + pos[5] = (bchdsr3 & FMC2_BCHDSR3_EBP6_MASK) >> FMC2_BCHDSR3_EBP6_SHIFT; + pos[6] = bchdsr4 & FMC2_BCHDSR4_EBP7_MASK; + pos[7] = (bchdsr4 & FMC2_BCHDSR4_EBP8_MASK) >> FMC2_BCHDSR4_EBP8_SHIFT; + + den = (bchdsr0 & FMC2_BCHDSR0_DEN_MASK) >> FMC2_BCHDSR0_DEN_SHIFT; + for (i = 0; i < den; i++) { + if (pos[i] < eccsize * 8) { + change_bit(pos[i], (unsigned long *)dat); + nb_errs++; + } + } + + return nb_errs; +} + +static int stm32_fmc2_bch_correct(struct nand_chip *chip, u8 *dat, + u8 *read_ecc, u8 *calc_ecc) +{ + struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller); + u32 ecc_sta[5]; + + /* Wait until the decoding error is ready */ + if (!wait_for_completion_timeout(&fmc2->complete, + msecs_to_jiffies(1000))) { + dev_err(fmc2->dev, "bch timeout\n"); + stm32_fmc2_disable_bch_irq(fmc2); + return -ETIMEDOUT; + } + + ecc_sta[0] = readl_relaxed(fmc2->io_base + FMC2_BCHDSR0); + ecc_sta[1] = readl_relaxed(fmc2->io_base + FMC2_BCHDSR1); + ecc_sta[2] = readl_relaxed(fmc2->io_base + FMC2_BCHDSR2); + ecc_sta[3] = readl_relaxed(fmc2->io_base + FMC2_BCHDSR3); + ecc_sta[4] = readl_relaxed(fmc2->io_base + FMC2_BCHDSR4); + + /* Disable ECC */ + stm32_fmc2_set_ecc(fmc2, false); + + return stm32_fmc2_bch_decode(chip->ecc.size, dat, ecc_sta); +} + +static int stm32_fmc2_read_page(struct nand_chip *chip, u8 *buf, + int oob_required, int page) +{ + struct mtd_info *mtd = nand_to_mtd(chip); + int ret, i, s, stat, eccsize = chip->ecc.size; + int eccbytes = chip->ecc.bytes; + int eccsteps = chip->ecc.steps; + int eccstrength = chip->ecc.strength; + u8 *p = buf; + u8 *ecc_calc = chip->ecc.calc_buf; + u8 *ecc_code = chip->ecc.code_buf; + unsigned int max_bitflips = 0; + + ret = nand_read_page_op(chip, page, 0, NULL, 0); + if (ret) + return ret; + + for (i = mtd->writesize + FMC2_BBM_LEN, s = 0; s < eccsteps; + s++, i += eccbytes, p += eccsize) { + chip->ecc.hwctl(chip, NAND_ECC_READ); + + /* Read the nand page sector (512 bytes) */ + ret = nand_change_read_column_op(chip, s * eccsize, p, + eccsize, false); + if (ret) + return ret; + + /* Read the corresponding ECC bytes */ + ret = nand_change_read_column_op(chip, i, ecc_code, + eccbytes, false); + if (ret) + return ret; + + /* Correct the data */ + stat = chip->ecc.correct(chip, p, ecc_code, ecc_calc); + if (stat == -EBADMSG) + /* Check for empty pages with bitflips */ + stat = nand_check_erased_ecc_chunk(p, eccsize, + ecc_code, eccbytes, + NULL, 0, + eccstrength); + + if (stat < 0) { + mtd->ecc_stats.failed++; + } else { + mtd->ecc_stats.corrected += stat; + max_bitflips = max_t(unsigned int, max_bitflips, stat); + } + } + + /* Read oob */ + if (oob_required) { + ret = nand_change_read_column_op(chip, mtd->writesize, + chip->oob_poi, mtd->oobsize, + false); + if (ret) + return ret; + } + + return max_bitflips; +} + +/* Sequencer read/write configuration */ +static void stm32_fmc2_rw_page_init(struct nand_chip *chip, int page, + int raw, bool write_data) +{ + struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller); + struct mtd_info *mtd = nand_to_mtd(chip); + u32 csqcfgr1, csqcfgr2, csqcfgr3; + u32 csqar1, csqar2; + u32 ecc_offset = mtd->writesize + FMC2_BBM_LEN; + u32 pcr = readl_relaxed(fmc2->io_base + FMC2_PCR); + + if (write_data) + pcr |= FMC2_PCR_WEN; + else + pcr &= ~FMC2_PCR_WEN; + writel_relaxed(pcr, fmc2->io_base + FMC2_PCR); + + /* + * - Set Program Page/Page Read command + * - Enable DMA request data + * - Set timings + */ + csqcfgr1 = FMC2_CSQCFGR1_DMADEN | FMC2_CSQCFGR1_CMD1T; + if (write_data) + csqcfgr1 |= FMC2_CSQCFGR1_CMD1(NAND_CMD_SEQIN); + else + csqcfgr1 |= FMC2_CSQCFGR1_CMD1(NAND_CMD_READ0) | + FMC2_CSQCFGR1_CMD2EN | + FMC2_CSQCFGR1_CMD2(NAND_CMD_READSTART) | + FMC2_CSQCFGR1_CMD2T; + + /* + * - Set Random Data Input/Random Data Read command + * - Enable the sequencer to access the Spare data area + * - Enable DMA request status decoding for read + * - Set timings + */ + if (write_data) + csqcfgr2 = FMC2_CSQCFGR2_RCMD1(NAND_CMD_RNDIN); + else + csqcfgr2 = FMC2_CSQCFGR2_RCMD1(NAND_CMD_RNDOUT) | + FMC2_CSQCFGR2_RCMD2EN | + FMC2_CSQCFGR2_RCMD2(NAND_CMD_RNDOUTSTART) | + FMC2_CSQCFGR2_RCMD1T | + FMC2_CSQCFGR2_RCMD2T; + if (!raw) { + csqcfgr2 |= write_data ? 0 : FMC2_CSQCFGR2_DMASEN; + csqcfgr2 |= FMC2_CSQCFGR2_SQSDTEN; + } + + /* + * - Set the number of sectors to be written + * - Set timings + */ + csqcfgr3 = FMC2_CSQCFGR3_SNBR(chip->ecc.steps - 1); + if (write_data) { + csqcfgr3 |= FMC2_CSQCFGR3_RAC2T; + if (chip->options & NAND_ROW_ADDR_3) + csqcfgr3 |= FMC2_CSQCFGR3_AC5T; + else + csqcfgr3 |= FMC2_CSQCFGR3_AC4T; + } + + /* + * Set the fourth first address cycles + * Byte 1 and byte 2 => column, we start at 0x0 + * Byte 3 and byte 4 => page + */ + csqar1 = FMC2_CSQCAR1_ADDC3(page); + csqar1 |= FMC2_CSQCAR1_ADDC4(page >> 8); + + /* + * - Set chip enable number + * - Set ECC byte offset in the spare area + * - Calculate the number of address cycles to be issued + * - Set byte 5 of address cycle if needed + */ + csqar2 = FMC2_CSQCAR2_NANDCEN(fmc2->cs_sel); + if (chip->options & NAND_BUSWIDTH_16) + csqar2 |= FMC2_CSQCAR2_SAO(ecc_offset >> 1); + else + csqar2 |= FMC2_CSQCAR2_SAO(ecc_offset); + if (chip->options & NAND_ROW_ADDR_3) { + csqcfgr1 |= FMC2_CSQCFGR1_ACYNBR(5); + csqar2 |= FMC2_CSQCAR2_ADDC5(page >> 16); + } else { + csqcfgr1 |= FMC2_CSQCFGR1_ACYNBR(4); + } + + writel_relaxed(csqcfgr1, fmc2->io_base + FMC2_CSQCFGR1); + writel_relaxed(csqcfgr2, fmc2->io_base + FMC2_CSQCFGR2); + writel_relaxed(csqcfgr3, fmc2->io_base + FMC2_CSQCFGR3); + writel_relaxed(csqar1, fmc2->io_base + FMC2_CSQAR1); + writel_relaxed(csqar2, fmc2->io_base + FMC2_CSQAR2); +} + +static void stm32_fmc2_dma_callback(void *arg) +{ + complete((struct completion *)arg); +} + +/* Read/write data from/to a page */ +static int stm32_fmc2_xfer(struct nand_chip *chip, const u8 *buf, + int raw, bool write_data) +{ + struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller); + struct dma_async_tx_descriptor *desc_data, *desc_ecc; + struct scatterlist *sg; + struct dma_chan *dma_ch = fmc2->dma_rx_ch; + enum dma_data_direction dma_data_dir = DMA_FROM_DEVICE; + enum dma_transfer_direction dma_transfer_dir = DMA_DEV_TO_MEM; + u32 csqcr = readl_relaxed(fmc2->io_base + FMC2_CSQCR); + int eccsteps = chip->ecc.steps; + int eccsize = chip->ecc.size; + const u8 *p = buf; + int s, ret; + + /* Configure DMA data */ + if (write_data) { + dma_data_dir = DMA_TO_DEVICE; + dma_transfer_dir = DMA_MEM_TO_DEV; + dma_ch = fmc2->dma_tx_ch; + } + + for_each_sg(fmc2->dma_data_sg.sgl, sg, eccsteps, s) { + sg_set_buf(sg, p, eccsize); + p += eccsize; + } + + ret = dma_map_sg(fmc2->dev, fmc2->dma_data_sg.sgl, + eccsteps, dma_data_dir); + if (ret < 0) + return ret; + + desc_data = dmaengine_prep_slave_sg(dma_ch, fmc2->dma_data_sg.sgl, + eccsteps, dma_transfer_dir, + DMA_PREP_INTERRUPT); + if (!desc_data) { + ret = -ENOMEM; + goto err_unmap_data; + } + + reinit_completion(&fmc2->dma_data_complete); + reinit_completion(&fmc2->complete); + desc_data->callback = stm32_fmc2_dma_callback; + desc_data->callback_param = &fmc2->dma_data_complete; + ret = dma_submit_error(dmaengine_submit(desc_data)); + if (ret) + goto err_unmap_data; + + dma_async_issue_pending(dma_ch); + + if (!write_data && !raw) { + /* Configure DMA ECC status */ + p = fmc2->ecc_buf; + for_each_sg(fmc2->dma_ecc_sg.sgl, sg, eccsteps, s) { + sg_set_buf(sg, p, fmc2->dma_ecc_len); + p += fmc2->dma_ecc_len; + } + + ret = dma_map_sg(fmc2->dev, fmc2->dma_ecc_sg.sgl, + eccsteps, dma_data_dir); + if (ret < 0) + goto err_unmap_data; + + desc_ecc = dmaengine_prep_slave_sg(fmc2->dma_ecc_ch, + fmc2->dma_ecc_sg.sgl, + eccsteps, dma_transfer_dir, + DMA_PREP_INTERRUPT); + if (!desc_ecc) { + ret = -ENOMEM; + goto err_unmap_ecc; + } + + reinit_completion(&fmc2->dma_ecc_complete); + desc_ecc->callback = stm32_fmc2_dma_callback; + desc_ecc->callback_param = &fmc2->dma_ecc_complete; + ret = dma_submit_error(dmaengine_submit(desc_ecc)); + if (ret) + goto err_unmap_ecc; + + dma_async_issue_pending(fmc2->dma_ecc_ch); + } + + stm32_fmc2_clear_seq_irq(fmc2); + stm32_fmc2_enable_seq_irq(fmc2); + + /* Start the transfer */ + csqcr |= FMC2_CSQCR_CSQSTART; + writel_relaxed(csqcr, fmc2->io_base + FMC2_CSQCR); + + /* Wait end of sequencer transfer */ + if (!wait_for_completion_timeout(&fmc2->complete, + msecs_to_jiffies(1000))) { + dev_err(fmc2->dev, "seq timeout\n"); + stm32_fmc2_disable_seq_irq(fmc2); + dmaengine_terminate_all(dma_ch); + if (!write_data && !raw) + dmaengine_terminate_all(fmc2->dma_ecc_ch); + ret = -ETIMEDOUT; + goto err_unmap_ecc; + } + + /* Wait DMA data transfer completion */ + if (!wait_for_completion_timeout(&fmc2->dma_data_complete, + msecs_to_jiffies(100))) { + dev_err(fmc2->dev, "data DMA timeout\n"); + dmaengine_terminate_all(dma_ch); + ret = -ETIMEDOUT; + } + + /* Wait DMA ECC transfer completion */ + if (!write_data && !raw) { + if (!wait_for_completion_timeout(&fmc2->dma_ecc_complete, + msecs_to_jiffies(100))) { + dev_err(fmc2->dev, "ECC DMA timeout\n"); + dmaengine_terminate_all(fmc2->dma_ecc_ch); + ret = -ETIMEDOUT; + } + } + +err_unmap_ecc: + if (!write_data && !raw) + dma_unmap_sg(fmc2->dev, fmc2->dma_ecc_sg.sgl, + eccsteps, dma_data_dir); + +err_unmap_data: + dma_unmap_sg(fmc2->dev, fmc2->dma_data_sg.sgl, eccsteps, dma_data_dir); + + return ret; +} + +static int stm32_fmc2_sequencer_write(struct nand_chip *chip, + const u8 *buf, int oob_required, + int page, int raw) +{ + struct mtd_info *mtd = nand_to_mtd(chip); + int ret; + + /* Configure the sequencer */ + stm32_fmc2_rw_page_init(chip, page, raw, true); + + /* Write the page */ + ret = stm32_fmc2_xfer(chip, buf, raw, true); + if (ret) + return ret; + + /* Write oob */ + if (oob_required) { + ret = nand_change_write_column_op(chip, mtd->writesize, + chip->oob_poi, mtd->oobsize, + false); + if (ret) + return ret; + } + + return nand_prog_page_end_op(chip); +} + +static int stm32_fmc2_sequencer_write_page(struct nand_chip *chip, + const u8 *buf, + int oob_required, + int page) +{ + int ret; + + /* Select the target */ + ret = stm32_fmc2_select_chip(chip, chip->cur_cs); + if (ret) + return ret; + + return stm32_fmc2_sequencer_write(chip, buf, oob_required, page, false); +} + +static int stm32_fmc2_sequencer_write_page_raw(struct nand_chip *chip, + const u8 *buf, + int oob_required, + int page) +{ + int ret; + + /* Select the target */ + ret = stm32_fmc2_select_chip(chip, chip->cur_cs); + if (ret) + return ret; + + return stm32_fmc2_sequencer_write(chip, buf, oob_required, page, true); +} + +/* Get a status indicating which sectors have errors */ +static inline u16 stm32_fmc2_get_mapping_status(struct stm32_fmc2_nfc *fmc2) +{ + u32 csqemsr = readl_relaxed(fmc2->io_base + FMC2_CSQEMSR); + + return csqemsr & FMC2_CSQEMSR_SEM; +} + +static int stm32_fmc2_sequencer_correct(struct nand_chip *chip, u8 *dat, + u8 *read_ecc, u8 *calc_ecc) +{ + struct mtd_info *mtd = nand_to_mtd(chip); + struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller); + int eccbytes = chip->ecc.bytes; + int eccsteps = chip->ecc.steps; + int eccstrength = chip->ecc.strength; + int i, s, eccsize = chip->ecc.size; + u32 *ecc_sta = (u32 *)fmc2->ecc_buf; + u16 sta_map = stm32_fmc2_get_mapping_status(fmc2); + unsigned int max_bitflips = 0; + + for (i = 0, s = 0; s < eccsteps; s++, i += eccbytes, dat += eccsize) { + int stat = 0; + + if (eccstrength == FMC2_ECC_HAM) { + /* Ecc_sta = FMC2_HECCR */ + if (sta_map & BIT(s)) { + stm32_fmc2_ham_set_ecc(*ecc_sta, &calc_ecc[i]); + stat = stm32_fmc2_ham_correct(chip, dat, + &read_ecc[i], + &calc_ecc[i]); + } + ecc_sta++; + } else { + /* + * Ecc_sta[0] = FMC2_BCHDSR0 + * Ecc_sta[1] = FMC2_BCHDSR1 + * Ecc_sta[2] = FMC2_BCHDSR2 + * Ecc_sta[3] = FMC2_BCHDSR3 + * Ecc_sta[4] = FMC2_BCHDSR4 + */ + if (sta_map & BIT(s)) + stat = stm32_fmc2_bch_decode(eccsize, dat, + ecc_sta); + ecc_sta += 5; + } + + if (stat == -EBADMSG) + /* Check for empty pages with bitflips */ + stat = nand_check_erased_ecc_chunk(dat, eccsize, + &read_ecc[i], + eccbytes, + NULL, 0, + eccstrength); + + if (stat < 0) { + mtd->ecc_stats.failed++; + } else { + mtd->ecc_stats.corrected += stat; + max_bitflips = max_t(unsigned int, max_bitflips, stat); + } + } + + return max_bitflips; +} + +static int stm32_fmc2_sequencer_read_page(struct nand_chip *chip, u8 *buf, + int oob_required, int page) +{ + struct mtd_info *mtd = nand_to_mtd(chip); + struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller); + u8 *ecc_calc = chip->ecc.calc_buf; + u8 *ecc_code = chip->ecc.code_buf; + u16 sta_map; + int ret; + + /* Select the target */ + ret = stm32_fmc2_select_chip(chip, chip->cur_cs); + if (ret) + return ret; + + /* Configure the sequencer */ + stm32_fmc2_rw_page_init(chip, page, 0, false); + + /* Read the page */ + ret = stm32_fmc2_xfer(chip, buf, 0, false); + if (ret) + return ret; + + sta_map = stm32_fmc2_get_mapping_status(fmc2); + + /* Check if errors happen */ + if (likely(!sta_map)) { + if (oob_required) + return nand_change_read_column_op(chip, mtd->writesize, + chip->oob_poi, + mtd->oobsize, false); + + return 0; + } + + /* Read oob */ + ret = nand_change_read_column_op(chip, mtd->writesize, + chip->oob_poi, mtd->oobsize, false); + if (ret) + return ret; + + ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0, + chip->ecc.total); + if (ret) + return ret; + + /* Correct data */ + return chip->ecc.correct(chip, buf, ecc_code, ecc_calc); +} + +static int stm32_fmc2_sequencer_read_page_raw(struct nand_chip *chip, u8 *buf, + int oob_required, int page) +{ + struct mtd_info *mtd = nand_to_mtd(chip); + int ret; + + /* Select the target */ + ret = stm32_fmc2_select_chip(chip, chip->cur_cs); + if (ret) + return ret; + + /* Configure the sequencer */ + stm32_fmc2_rw_page_init(chip, page, 1, false); + + /* Read the page */ + ret = stm32_fmc2_xfer(chip, buf, 1, false); + if (ret) + return ret; + + /* Read oob */ + if (oob_required) + return nand_change_read_column_op(chip, mtd->writesize, + chip->oob_poi, mtd->oobsize, + false); + + return 0; +} + +static irqreturn_t stm32_fmc2_irq(int irq, void *dev_id) +{ + struct stm32_fmc2_nfc *fmc2 = (struct stm32_fmc2_nfc *)dev_id; + + if (fmc2->irq_state == FMC2_IRQ_SEQ) + /* Sequencer is used */ + stm32_fmc2_disable_seq_irq(fmc2); + else if (fmc2->irq_state == FMC2_IRQ_BCH) + /* BCH is used */ + stm32_fmc2_disable_bch_irq(fmc2); + + complete(&fmc2->complete); + + return IRQ_HANDLED; +} + +static void stm32_fmc2_read_data(struct nand_chip *chip, void *buf, + unsigned int len, bool force_8bit) +{ + struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller); + void __iomem *io_addr_r = fmc2->data_base[fmc2->cs_sel]; + + if (force_8bit && chip->options & NAND_BUSWIDTH_16) + /* Reconfigure bus width to 8-bit */ + stm32_fmc2_set_buswidth_16(fmc2, false); + + if (!IS_ALIGNED((uintptr_t)buf, sizeof(u32))) { + if (!IS_ALIGNED((uintptr_t)buf, sizeof(u16)) && len) { + *(u8 *)buf = readb_relaxed(io_addr_r); + buf += sizeof(u8); + len -= sizeof(u8); + } + + if (!IS_ALIGNED((uintptr_t)buf, sizeof(u32)) && + len >= sizeof(u16)) { + *(u16 *)buf = readw_relaxed(io_addr_r); + buf += sizeof(u16); + len -= sizeof(u16); + } + } + + /* Buf is aligned */ + while (len >= sizeof(u32)) { + *(u32 *)buf = readl_relaxed(io_addr_r); + buf += sizeof(u32); + len -= sizeof(u32); + } + + /* Read remaining bytes */ + if (len >= sizeof(u16)) { + *(u16 *)buf = readw_relaxed(io_addr_r); + buf += sizeof(u16); + len -= sizeof(u16); + } + + if (len) + *(u8 *)buf = readb_relaxed(io_addr_r); + + if (force_8bit && chip->options & NAND_BUSWIDTH_16) + /* Reconfigure bus width to 16-bit */ + stm32_fmc2_set_buswidth_16(fmc2, true); +} + +static void stm32_fmc2_write_data(struct nand_chip *chip, const void *buf, + unsigned int len, bool force_8bit) +{ + struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller); + void __iomem *io_addr_w = fmc2->data_base[fmc2->cs_sel]; + + if (force_8bit && chip->options & NAND_BUSWIDTH_16) + /* Reconfigure bus width to 8-bit */ + stm32_fmc2_set_buswidth_16(fmc2, false); + + if (!IS_ALIGNED((uintptr_t)buf, sizeof(u32))) { + if (!IS_ALIGNED((uintptr_t)buf, sizeof(u16)) && len) { + writeb_relaxed(*(u8 *)buf, io_addr_w); + buf += sizeof(u8); + len -= sizeof(u8); + } + + if (!IS_ALIGNED((uintptr_t)buf, sizeof(u32)) && + len >= sizeof(u16)) { + writew_relaxed(*(u16 *)buf, io_addr_w); + buf += sizeof(u16); + len -= sizeof(u16); + } + } + + /* Buf is aligned */ + while (len >= sizeof(u32)) { + writel_relaxed(*(u32 *)buf, io_addr_w); + buf += sizeof(u32); + len -= sizeof(u32); + } + + /* Write remaining bytes */ + if (len >= sizeof(u16)) { + writew_relaxed(*(u16 *)buf, io_addr_w); + buf += sizeof(u16); + len -= sizeof(u16); + } + + if (len) + writeb_relaxed(*(u8 *)buf, io_addr_w); + + if (force_8bit && chip->options & NAND_BUSWIDTH_16) + /* Reconfigure bus width to 16-bit */ + stm32_fmc2_set_buswidth_16(fmc2, true); +} + +static int stm32_fmc2_exec_op(struct nand_chip *chip, + const struct nand_operation *op, + bool check_only) +{ + struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller); + const struct nand_op_instr *instr = NULL; + unsigned int op_id, i; + int ret; + + ret = stm32_fmc2_select_chip(chip, op->cs); + if (ret) + return ret; + + if (check_only) + return ret; + + for (op_id = 0; op_id < op->ninstrs; op_id++) { + instr = &op->instrs[op_id]; + + switch (instr->type) { + case NAND_OP_CMD_INSTR: + writeb_relaxed(instr->ctx.cmd.opcode, + fmc2->cmd_base[fmc2->cs_sel]); + break; + + case NAND_OP_ADDR_INSTR: + for (i = 0; i < instr->ctx.addr.naddrs; i++) + writeb_relaxed(instr->ctx.addr.addrs[i], + fmc2->addr_base[fmc2->cs_sel]); + break; + + case NAND_OP_DATA_IN_INSTR: + stm32_fmc2_read_data(chip, instr->ctx.data.buf.in, + instr->ctx.data.len, + instr->ctx.data.force_8bit); + break; + + case NAND_OP_DATA_OUT_INSTR: + stm32_fmc2_write_data(chip, instr->ctx.data.buf.out, + instr->ctx.data.len, + instr->ctx.data.force_8bit); + break; + + case NAND_OP_WAITRDY_INSTR: + ret = nand_soft_waitrdy(chip, + instr->ctx.waitrdy.timeout_ms); + break; + } + } + + return ret; +} + +/* Controller initialization */ +static void stm32_fmc2_init(struct stm32_fmc2_nfc *fmc2) +{ + u32 pcr = readl_relaxed(fmc2->io_base + FMC2_PCR); + u32 bcr1 = readl_relaxed(fmc2->io_base + FMC2_BCR1); + + /* Set CS used to undefined */ + fmc2->cs_sel = -1; + + /* Enable wait feature and nand flash memory bank */ + pcr |= FMC2_PCR_PWAITEN; + pcr |= FMC2_PCR_PBKEN; + + /* Set buswidth to 8 bits mode for identification */ + pcr &= ~FMC2_PCR_PWID_MASK; + + /* ECC logic is disabled */ + pcr &= ~FMC2_PCR_ECCEN; + + /* Default mode */ + pcr &= ~FMC2_PCR_ECCALG; + pcr &= ~FMC2_PCR_BCHECC; + pcr &= ~FMC2_PCR_WEN; + + /* Set default ECC sector size */ + pcr &= ~FMC2_PCR_ECCSS_MASK; + pcr |= FMC2_PCR_ECCSS(FMC2_PCR_ECCSS_2048); + + /* Set default tclr/tar timings */ + pcr &= ~FMC2_PCR_TCLR_MASK; + pcr |= FMC2_PCR_TCLR(FMC2_PCR_TCLR_DEFAULT); + pcr &= ~FMC2_PCR_TAR_MASK; + pcr |= FMC2_PCR_TAR(FMC2_PCR_TAR_DEFAULT); + + /* Enable FMC2 controller */ + bcr1 |= FMC2_BCR1_FMC2EN; + + writel_relaxed(bcr1, fmc2->io_base + FMC2_BCR1); + writel_relaxed(pcr, fmc2->io_base + FMC2_PCR); + writel_relaxed(FMC2_PMEM_DEFAULT, fmc2->io_base + FMC2_PMEM); + writel_relaxed(FMC2_PATT_DEFAULT, fmc2->io_base + FMC2_PATT); +} + +/* Controller timings */ +static void stm32_fmc2_calc_timings(struct nand_chip *chip, + const struct nand_sdr_timings *sdrt) +{ + struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller); + struct stm32_fmc2_nand *nand = to_fmc2_nand(chip); + struct stm32_fmc2_timings *tims = &nand->timings; + unsigned long hclk = clk_get_rate(fmc2->clk); + unsigned long hclkp = NSEC_PER_SEC / (hclk / 1000); + int tar, tclr, thiz, twait, tset_mem, tset_att, thold_mem, thold_att; + + tar = hclkp; + if (tar < sdrt->tAR_min) + tar = sdrt->tAR_min; + tims->tar = DIV_ROUND_UP(tar, hclkp) - 1; + if (tims->tar > FMC2_PCR_TIMING_MASK) + tims->tar = FMC2_PCR_TIMING_MASK; + + tclr = hclkp; + if (tclr < sdrt->tCLR_min) + tclr = sdrt->tCLR_min; + tims->tclr = DIV_ROUND_UP(tclr, hclkp) - 1; + if (tims->tclr > FMC2_PCR_TIMING_MASK) + tims->tclr = FMC2_PCR_TIMING_MASK; + + tims->thiz = FMC2_THIZ; + thiz = (tims->thiz + 1) * hclkp; + + /* + * tWAIT > tRP + * tWAIT > tWP + * tWAIT > tREA + tIO + */ + twait = hclkp; + if (twait < sdrt->tRP_min) + twait = sdrt->tRP_min; + if (twait < sdrt->tWP_min) + twait = sdrt->tWP_min; + if (twait < sdrt->tREA_max + FMC2_TIO) + twait = sdrt->tREA_max + FMC2_TIO; + tims->twait = DIV_ROUND_UP(twait, hclkp); + if (tims->twait == 0) + tims->twait = 1; + else if (tims->twait > FMC2_PMEM_PATT_TIMING_MASK) + tims->twait = FMC2_PMEM_PATT_TIMING_MASK; + + /* + * tSETUP_MEM > tCS - tWAIT + * tSETUP_MEM > tALS - tWAIT + * tSETUP_MEM > tDS - (tWAIT - tHIZ) + */ + tset_mem = hclkp; + if (sdrt->tCS_min > twait && (tset_mem < sdrt->tCS_min - twait)) + tset_mem = sdrt->tCS_min - twait; + if (sdrt->tALS_min > twait && (tset_mem < sdrt->tALS_min - twait)) + tset_mem = sdrt->tALS_min - twait; + if (twait > thiz && (sdrt->tDS_min > twait - thiz) && + (tset_mem < sdrt->tDS_min - (twait - thiz))) + tset_mem = sdrt->tDS_min - (twait - thiz); + tims->tset_mem = DIV_ROUND_UP(tset_mem, hclkp); + if (tims->tset_mem == 0) + tims->tset_mem = 1; + else if (tims->tset_mem > FMC2_PMEM_PATT_TIMING_MASK) + tims->tset_mem = FMC2_PMEM_PATT_TIMING_MASK; + + /* + * tHOLD_MEM > tCH + * tHOLD_MEM > tREH - tSETUP_MEM + * tHOLD_MEM > max(tRC, tWC) - (tSETUP_MEM + tWAIT) + */ + thold_mem = hclkp; + if (thold_mem < sdrt->tCH_min) + thold_mem = sdrt->tCH_min; + if (sdrt->tREH_min > tset_mem && + (thold_mem < sdrt->tREH_min - tset_mem)) + thold_mem = sdrt->tREH_min - tset_mem; + if ((sdrt->tRC_min > tset_mem + twait) && + (thold_mem < sdrt->tRC_min - (tset_mem + twait))) + thold_mem = sdrt->tRC_min - (tset_mem + twait); + if ((sdrt->tWC_min > tset_mem + twait) && + (thold_mem < sdrt->tWC_min - (tset_mem + twait))) + thold_mem = sdrt->tWC_min - (tset_mem + twait); + tims->thold_mem = DIV_ROUND_UP(thold_mem, hclkp); + if (tims->thold_mem == 0) + tims->thold_mem = 1; + else if (tims->thold_mem > FMC2_PMEM_PATT_TIMING_MASK) + tims->thold_mem = FMC2_PMEM_PATT_TIMING_MASK; + + /* + * tSETUP_ATT > tCS - tWAIT + * tSETUP_ATT > tCLS - tWAIT + * tSETUP_ATT > tALS - tWAIT + * tSETUP_ATT > tRHW - tHOLD_MEM + * tSETUP_ATT > tDS - (tWAIT - tHIZ) + */ + tset_att = hclkp; + if (sdrt->tCS_min > twait && (tset_att < sdrt->tCS_min - twait)) + tset_att = sdrt->tCS_min - twait; + if (sdrt->tCLS_min > twait && (tset_att < sdrt->tCLS_min - twait)) + tset_att = sdrt->tCLS_min - twait; + if (sdrt->tALS_min > twait && (tset_att < sdrt->tALS_min - twait)) + tset_att = sdrt->tALS_min - twait; + if (sdrt->tRHW_min > thold_mem && + (tset_att < sdrt->tRHW_min - thold_mem)) + tset_att = sdrt->tRHW_min - thold_mem; + if (twait > thiz && (sdrt->tDS_min > twait - thiz) && + (tset_att < sdrt->tDS_min - (twait - thiz))) + tset_att = sdrt->tDS_min - (twait - thiz); + tims->tset_att = DIV_ROUND_UP(tset_att, hclkp); + if (tims->tset_att == 0) + tims->tset_att = 1; + else if (tims->tset_att > FMC2_PMEM_PATT_TIMING_MASK) + tims->tset_att = FMC2_PMEM_PATT_TIMING_MASK; + + /* + * tHOLD_ATT > tALH + * tHOLD_ATT > tCH + * tHOLD_ATT > tCLH + * tHOLD_ATT > tCOH + * tHOLD_ATT > tDH + * tHOLD_ATT > tWB + tIO + tSYNC - tSETUP_MEM + * tHOLD_ATT > tADL - tSETUP_MEM + * tHOLD_ATT > tWH - tSETUP_MEM + * tHOLD_ATT > tWHR - tSETUP_MEM + * tHOLD_ATT > tRC - (tSETUP_ATT + tWAIT) + * tHOLD_ATT > tWC - (tSETUP_ATT + tWAIT) + */ + thold_att = hclkp; + if (thold_att < sdrt->tALH_min) + thold_att = sdrt->tALH_min; + if (thold_att < sdrt->tCH_min) + thold_att = sdrt->tCH_min; + if (thold_att < sdrt->tCLH_min) + thold_att = sdrt->tCLH_min; + if (thold_att < sdrt->tCOH_min) + thold_att = sdrt->tCOH_min; + if (thold_att < sdrt->tDH_min) + thold_att = sdrt->tDH_min; + if ((sdrt->tWB_max + FMC2_TIO + FMC2_TSYNC > tset_mem) && + (thold_att < sdrt->tWB_max + FMC2_TIO + FMC2_TSYNC - tset_mem)) + thold_att = sdrt->tWB_max + FMC2_TIO + FMC2_TSYNC - tset_mem; + if (sdrt->tADL_min > tset_mem && + (thold_att < sdrt->tADL_min - tset_mem)) + thold_att = sdrt->tADL_min - tset_mem; + if (sdrt->tWH_min > tset_mem && + (thold_att < sdrt->tWH_min - tset_mem)) + thold_att = sdrt->tWH_min - tset_mem; + if (sdrt->tWHR_min > tset_mem && + (thold_att < sdrt->tWHR_min - tset_mem)) + thold_att = sdrt->tWHR_min - tset_mem; + if ((sdrt->tRC_min > tset_att + twait) && + (thold_att < sdrt->tRC_min - (tset_att + twait))) + thold_att = sdrt->tRC_min - (tset_att + twait); + if ((sdrt->tWC_min > tset_att + twait) && + (thold_att < sdrt->tWC_min - (tset_att + twait))) + thold_att = sdrt->tWC_min - (tset_att + twait); + tims->thold_att = DIV_ROUND_UP(thold_att, hclkp); + if (tims->thold_att == 0) + tims->thold_att = 1; + else if (tims->thold_att > FMC2_PMEM_PATT_TIMING_MASK) + tims->thold_att = FMC2_PMEM_PATT_TIMING_MASK; +} + +static int stm32_fmc2_setup_interface(struct nand_chip *chip, int chipnr, + const struct nand_data_interface *conf) +{ + const struct nand_sdr_timings *sdrt; + + sdrt = nand_get_sdr_timings(conf); + if (IS_ERR(sdrt)) + return PTR_ERR(sdrt); + + if (chipnr == NAND_DATA_IFACE_CHECK_ONLY) + return 0; + + stm32_fmc2_calc_timings(chip, sdrt); + + /* Apply timings */ + stm32_fmc2_timings_init(chip); + + return 0; +} + +/* DMA configuration */ +static int stm32_fmc2_dma_setup(struct stm32_fmc2_nfc *fmc2) +{ + int ret; + + fmc2->dma_tx_ch = dma_request_slave_channel(fmc2->dev, "tx"); + fmc2->dma_rx_ch = dma_request_slave_channel(fmc2->dev, "rx"); + fmc2->dma_ecc_ch = dma_request_slave_channel(fmc2->dev, "ecc"); + + if (!fmc2->dma_tx_ch || !fmc2->dma_rx_ch || !fmc2->dma_ecc_ch) { + dev_warn(fmc2->dev, "DMAs not defined in the device tree, polling mode is used\n"); + return 0; + } + + ret = sg_alloc_table(&fmc2->dma_ecc_sg, FMC2_MAX_SG, GFP_KERNEL); + if (ret) + return ret; + + /* Allocate a buffer to store ECC status registers */ + fmc2->ecc_buf = devm_kzalloc(fmc2->dev, FMC2_MAX_ECC_BUF_LEN, + GFP_KERNEL); + if (!fmc2->ecc_buf) + return -ENOMEM; + + ret = sg_alloc_table(&fmc2->dma_data_sg, FMC2_MAX_SG, GFP_KERNEL); + if (ret) + return ret; + + init_completion(&fmc2->dma_data_complete); + init_completion(&fmc2->dma_ecc_complete); + + return 0; +} + +/* NAND callbacks setup */ +static void stm32_fmc2_nand_callbacks_setup(struct nand_chip *chip) +{ + struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller); + + /* + * Specific callbacks to read/write a page depending on + * the mode (polling/sequencer) and the algo used (Hamming, BCH). + */ + if (fmc2->dma_tx_ch && fmc2->dma_rx_ch && fmc2->dma_ecc_ch) { + /* DMA => use sequencer mode callbacks */ + chip->ecc.correct = stm32_fmc2_sequencer_correct; + chip->ecc.write_page = stm32_fmc2_sequencer_write_page; + chip->ecc.read_page = stm32_fmc2_sequencer_read_page; + chip->ecc.write_page_raw = stm32_fmc2_sequencer_write_page_raw; + chip->ecc.read_page_raw = stm32_fmc2_sequencer_read_page_raw; + } else { + /* No DMA => use polling mode callbacks */ + chip->ecc.hwctl = stm32_fmc2_hwctl; + if (chip->ecc.strength == FMC2_ECC_HAM) { + /* Hamming is used */ + chip->ecc.calculate = stm32_fmc2_ham_calculate; + chip->ecc.correct = stm32_fmc2_ham_correct; + chip->ecc.options |= NAND_ECC_GENERIC_ERASED_CHECK; + } else { + /* BCH is used */ + chip->ecc.calculate = stm32_fmc2_bch_calculate; + chip->ecc.correct = stm32_fmc2_bch_correct; + chip->ecc.read_page = stm32_fmc2_read_page; + } + } + + /* Specific configurations depending on the algo used */ + if (chip->ecc.strength == FMC2_ECC_HAM) + chip->ecc.bytes = chip->options & NAND_BUSWIDTH_16 ? 4 : 3; + else if (chip->ecc.strength == FMC2_ECC_BCH8) + chip->ecc.bytes = chip->options & NAND_BUSWIDTH_16 ? 14 : 13; + else + chip->ecc.bytes = chip->options & NAND_BUSWIDTH_16 ? 8 : 7; +} + +/* FMC2 layout */ +static int stm32_fmc2_nand_ooblayout_ecc(struct mtd_info *mtd, int section, + struct mtd_oob_region *oobregion) +{ + struct nand_chip *chip = mtd_to_nand(mtd); + struct nand_ecc_ctrl *ecc = &chip->ecc; + + if (section) + return -ERANGE; + + oobregion->length = ecc->total; + oobregion->offset = FMC2_BBM_LEN; + + return 0; +} + +static int stm32_fmc2_nand_ooblayout_free(struct mtd_info *mtd, int section, + struct mtd_oob_region *oobregion) +{ + struct nand_chip *chip = mtd_to_nand(mtd); + struct nand_ecc_ctrl *ecc = &chip->ecc; + + if (section) + return -ERANGE; + + oobregion->length = mtd->oobsize - ecc->total - FMC2_BBM_LEN; + oobregion->offset = ecc->total + FMC2_BBM_LEN; + + return 0; +} + +static const struct mtd_ooblayout_ops stm32_fmc2_nand_ooblayout_ops = { + .ecc = stm32_fmc2_nand_ooblayout_ecc, + .free = stm32_fmc2_nand_ooblayout_free, +}; + +/* FMC2 caps */ +static int stm32_fmc2_calc_ecc_bytes(int step_size, int strength) +{ + /* Hamming */ + if (strength == FMC2_ECC_HAM) + return 4; + + /* BCH8 */ + if (strength == FMC2_ECC_BCH8) + return 14; + + /* BCH4 */ + return 8; +} + +NAND_ECC_CAPS_SINGLE(stm32_fmc2_ecc_caps, stm32_fmc2_calc_ecc_bytes, + FMC2_ECC_STEP_SIZE, + FMC2_ECC_HAM, FMC2_ECC_BCH4, FMC2_ECC_BCH8); + +/* FMC2 controller ops */ +static int stm32_fmc2_attach_chip(struct nand_chip *chip) +{ + struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller); + struct mtd_info *mtd = nand_to_mtd(chip); + int ret; + + /* + * Only NAND_ECC_HW mode is actually supported + * Hamming => ecc.strength = 1 + * BCH4 => ecc.strength = 4 + * BCH8 => ecc.strength = 8 + * ECC sector size = 512 + */ + if (chip->ecc.mode != NAND_ECC_HW) { + dev_err(fmc2->dev, "nand_ecc_mode is not well defined in the DT\n"); + return -EINVAL; + } + + ret = nand_ecc_choose_conf(chip, &stm32_fmc2_ecc_caps, + mtd->oobsize - FMC2_BBM_LEN); + if (ret) { + dev_err(fmc2->dev, "no valid ECC settings set\n"); + return ret; + } + + if (mtd->writesize / chip->ecc.size > FMC2_MAX_SG) { + dev_err(fmc2->dev, "nand page size is not supported\n"); + return -EINVAL; + } + + if (chip->bbt_options & NAND_BBT_USE_FLASH) + chip->bbt_options |= NAND_BBT_NO_OOB; + + /* NAND callbacks setup */ + stm32_fmc2_nand_callbacks_setup(chip); + + /* Define ECC layout */ + mtd_set_ooblayout(mtd, &stm32_fmc2_nand_ooblayout_ops); + + /* Configure bus width to 16-bit */ + if (chip->options & NAND_BUSWIDTH_16) + stm32_fmc2_set_buswidth_16(fmc2, true); + + return 0; +} + +static const struct nand_controller_ops stm32_fmc2_nand_controller_ops = { + .attach_chip = stm32_fmc2_attach_chip, + .exec_op = stm32_fmc2_exec_op, + .setup_data_interface = stm32_fmc2_setup_interface, +}; + +/* FMC2 probe */ +static int stm32_fmc2_parse_child(struct stm32_fmc2_nfc *fmc2, + struct device_node *dn) +{ + struct stm32_fmc2_nand *nand = &fmc2->nand; + u32 cs; + int ret, i; + + if (!of_get_property(dn, "reg", &nand->ncs)) + return -EINVAL; + + nand->ncs /= sizeof(u32); + if (!nand->ncs) { + dev_err(fmc2->dev, "invalid reg property size\n"); + return -EINVAL; + } + + for (i = 0; i < nand->ncs; i++) { + ret = of_property_read_u32_index(dn, "reg", i, &cs); + if (ret) { + dev_err(fmc2->dev, "could not retrieve reg property: %d\n", + ret); + return ret; + } + + if (cs > FMC2_MAX_CE) { + dev_err(fmc2->dev, "invalid reg value: %d\n", cs); + return -EINVAL; + } + + if (fmc2->cs_assigned & BIT(cs)) { + dev_err(fmc2->dev, "cs already assigned: %d\n", cs); + return -EINVAL; + } + + fmc2->cs_assigned |= BIT(cs); + nand->cs_used[i] = cs; + } + + nand_set_flash_node(&nand->chip, dn); + + return 0; +} + +static int stm32_fmc2_parse_dt(struct stm32_fmc2_nfc *fmc2) +{ + struct device_node *dn = fmc2->dev->of_node; + struct device_node *child; + int nchips = of_get_child_count(dn); + int ret = 0; + + if (!nchips) { + dev_err(fmc2->dev, "NAND chip not defined\n"); + return -EINVAL; + } + + if (nchips > 1) { + dev_err(fmc2->dev, "too many NAND chips defined\n"); + return -EINVAL; + } + + for_each_child_of_node(dn, child) { + ret = stm32_fmc2_parse_child(fmc2, child); + if (ret < 0) { + of_node_put(child); + return ret; + } + } + + return ret; +} + +static int stm32_fmc2_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct reset_control *rstc; + struct stm32_fmc2_nfc *fmc2; + struct stm32_fmc2_nand *nand; + struct resource *res; + struct mtd_info *mtd; + struct nand_chip *chip; + int chip_cs, mem_region, ret, irq; + + fmc2 = devm_kzalloc(dev, sizeof(*fmc2), GFP_KERNEL); + if (!fmc2) + return -ENOMEM; + + fmc2->dev = dev; + nand_controller_init(&fmc2->base); + fmc2->base.ops = &stm32_fmc2_nand_controller_ops; + + ret = stm32_fmc2_parse_dt(fmc2); + if (ret) + return ret; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + fmc2->io_base = devm_ioremap_resource(dev, res); + if (IS_ERR(fmc2->io_base)) + return PTR_ERR(fmc2->io_base); + + fmc2->io_phys_addr = res->start; + + for (chip_cs = 0, mem_region = 1; chip_cs < FMC2_MAX_CE; + chip_cs++, mem_region += 3) { + if (!(fmc2->cs_assigned & BIT(chip_cs))) + continue; + + res = platform_get_resource(pdev, IORESOURCE_MEM, mem_region); + fmc2->data_base[chip_cs] = devm_ioremap_resource(dev, res); + if (IS_ERR(fmc2->data_base[chip_cs])) + return PTR_ERR(fmc2->data_base[chip_cs]); + + fmc2->data_phys_addr[chip_cs] = res->start; + + res = platform_get_resource(pdev, IORESOURCE_MEM, + mem_region + 1); + fmc2->cmd_base[chip_cs] = devm_ioremap_resource(dev, res); + if (IS_ERR(fmc2->cmd_base[chip_cs])) + return PTR_ERR(fmc2->cmd_base[chip_cs]); + + res = platform_get_resource(pdev, IORESOURCE_MEM, + mem_region + 2); + fmc2->addr_base[chip_cs] = devm_ioremap_resource(dev, res); + if (IS_ERR(fmc2->addr_base[chip_cs])) + return PTR_ERR(fmc2->addr_base[chip_cs]); + } + + irq = platform_get_irq(pdev, 0); + ret = devm_request_irq(dev, irq, stm32_fmc2_irq, 0, + dev_name(dev), fmc2); + if (ret) { + dev_err(dev, "failed to request irq\n"); + return ret; + } + + init_completion(&fmc2->complete); + + fmc2->clk = devm_clk_get(dev, NULL); + if (IS_ERR(fmc2->clk)) + return PTR_ERR(fmc2->clk); + + ret = clk_prepare_enable(fmc2->clk); + if (ret) { + dev_err(dev, "can not enable the clock\n"); + return ret; + } + + rstc = devm_reset_control_get(dev, NULL); + if (!IS_ERR(rstc)) { + reset_control_assert(rstc); + reset_control_deassert(rstc); + } + + /* DMA setup */ + ret = stm32_fmc2_dma_setup(fmc2); + if (ret) + return ret; + + /* FMC2 init routine */ + stm32_fmc2_init(fmc2); + + nand = &fmc2->nand; + chip = &nand->chip; + mtd = nand_to_mtd(chip); + mtd->dev.parent = dev; + + chip->controller = &fmc2->base; + chip->options |= NAND_BUSWIDTH_AUTO | NAND_NO_SUBPAGE_WRITE | + NAND_USE_BOUNCE_BUFFER; + + /* Default ECC settings */ + chip->ecc.mode = NAND_ECC_HW; + chip->ecc.size = FMC2_ECC_STEP_SIZE; + chip->ecc.strength = FMC2_ECC_BCH8; + + /* Scan to find existence of the device */ + ret = nand_scan(chip, nand->ncs); + if (ret) + goto err_scan; + + ret = mtd_device_register(mtd, NULL, 0); + if (ret) + goto err_device_register; + + platform_set_drvdata(pdev, fmc2); + + return 0; + +err_device_register: + nand_cleanup(chip); + +err_scan: + if (fmc2->dma_ecc_ch) + dma_release_channel(fmc2->dma_ecc_ch); + if (fmc2->dma_tx_ch) + dma_release_channel(fmc2->dma_tx_ch); + if (fmc2->dma_rx_ch) + dma_release_channel(fmc2->dma_rx_ch); + + sg_free_table(&fmc2->dma_data_sg); + sg_free_table(&fmc2->dma_ecc_sg); + + clk_disable_unprepare(fmc2->clk); + + return ret; +} + +static int stm32_fmc2_remove(struct platform_device *pdev) +{ + struct stm32_fmc2_nfc *fmc2 = platform_get_drvdata(pdev); + struct stm32_fmc2_nand *nand = &fmc2->nand; + + nand_release(&nand->chip); + + if (fmc2->dma_ecc_ch) + dma_release_channel(fmc2->dma_ecc_ch); + if (fmc2->dma_tx_ch) + dma_release_channel(fmc2->dma_tx_ch); + if (fmc2->dma_rx_ch) + dma_release_channel(fmc2->dma_rx_ch); + + sg_free_table(&fmc2->dma_data_sg); + sg_free_table(&fmc2->dma_ecc_sg); + + clk_disable_unprepare(fmc2->clk); + + return 0; +} + +static int __maybe_unused stm32_fmc2_suspend(struct device *dev) +{ + struct stm32_fmc2_nfc *fmc2 = dev_get_drvdata(dev); + + clk_disable_unprepare(fmc2->clk); + + pinctrl_pm_select_sleep_state(dev); + + return 0; +} + +static int __maybe_unused stm32_fmc2_resume(struct device *dev) +{ + struct stm32_fmc2_nfc *fmc2 = dev_get_drvdata(dev); + struct stm32_fmc2_nand *nand = &fmc2->nand; + int chip_cs, ret; + + pinctrl_pm_select_default_state(dev); + + ret = clk_prepare_enable(fmc2->clk); + if (ret) { + dev_err(dev, "can not enable the clock\n"); + return ret; + } + + stm32_fmc2_init(fmc2); + + for (chip_cs = 0; chip_cs < FMC2_MAX_CE; chip_cs++) { + if (!(fmc2->cs_assigned & BIT(chip_cs))) + continue; + + nand_reset(&nand->chip, chip_cs); + } + + return 0; +} + +static SIMPLE_DEV_PM_OPS(stm32_fmc2_pm_ops, stm32_fmc2_suspend, + stm32_fmc2_resume); + +static const struct of_device_id stm32_fmc2_match[] = { + {.compatible = "st,stm32mp15-fmc2"}, + {} +}; +MODULE_DEVICE_TABLE(of, stm32_fmc2_match); + +static struct platform_driver stm32_fmc2_driver = { + .probe = stm32_fmc2_probe, + .remove = stm32_fmc2_remove, + .driver = { + .name = "stm32_fmc2_nand", + .of_match_table = stm32_fmc2_match, + .pm = &stm32_fmc2_pm_ops, + }, +}; +module_platform_driver(stm32_fmc2_driver); + +MODULE_ALIAS("platform:stm32_fmc2_nand"); +MODULE_AUTHOR("Christophe Kerello <christophe.kerello@st.com>"); +MODULE_DESCRIPTION("STMicroelectronics STM32 FMC2 nand driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/mtd/nand/raw/sunxi_nand.c b/drivers/mtd/nand/raw/sunxi_nand.c index e828ee50a201..4282bc477761 100644 --- a/drivers/mtd/nand/raw/sunxi_nand.c +++ b/drivers/mtd/nand/raw/sunxi_nand.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0+ /* * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com> * @@ -10,16 +11,6 @@ * * Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com> * Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include <linux/dma-mapping.h> @@ -163,38 +154,36 @@ #define NFC_MAX_CS 7 -/* - * Chip Select structure: stores information related to NAND Chip Select +/** + * struct sunxi_nand_chip_sel - stores information related to NAND Chip Select * - * @cs: the NAND CS id used to communicate with a NAND Chip - * @rb: the Ready/Busy pin ID. -1 means no R/B pin connected to the - * NFC + * @cs: the NAND CS id used to communicate with a NAND Chip + * @rb: the Ready/Busy pin ID. -1 means no R/B pin connected to the NFC */ struct sunxi_nand_chip_sel { u8 cs; s8 rb; }; -/* - * sunxi HW ECC infos: stores information related to HW ECC support +/** + * struct sunxi_nand_hw_ecc - stores information related to HW ECC support * - * @mode: the sunxi ECC mode field deduced from ECC requirements + * @mode: the sunxi ECC mode field deduced from ECC requirements */ struct sunxi_nand_hw_ecc { int mode; }; -/* - * NAND chip structure: stores NAND chip device related information +/** + * struct sunxi_nand_chip - stores NAND chip device related information * - * @node: used to store NAND chips into a list - * @nand: base NAND chip structure - * @mtd: base MTD structure - * @clk_rate: clk_rate required for this NAND chip - * @timing_cfg TIMING_CFG register value for this NAND chip - * @selected: current active CS - * @nsels: number of CS lines required by the NAND chip - * @sels: array of CS lines descriptions + * @node: used to store NAND chips into a list + * @nand: base NAND chip structure + * @clk_rate: clk_rate required for this NAND chip + * @timing_cfg: TIMING_CFG register value for this NAND chip + * @timing_ctl: TIMING_CTL register value for this NAND chip + * @nsels: number of CS lines required by the NAND chip + * @sels: array of CS lines descriptions */ struct sunxi_nand_chip { struct list_head node; @@ -202,11 +191,6 @@ struct sunxi_nand_chip { unsigned long clk_rate; u32 timing_cfg; u32 timing_ctl; - int selected; - int addr_cycles; - u32 addr[2]; - int cmd_cycles; - u8 cmd[2]; int nsels; struct sunxi_nand_chip_sel sels[0]; }; @@ -216,20 +200,21 @@ static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand) return container_of(nand, struct sunxi_nand_chip, nand); } -/* - * NAND Controller structure: stores sunxi NAND controller information +/** + * struct sunxi_nfc - stores sunxi NAND controller information * - * @controller: base controller structure - * @dev: parent device (used to print error messages) - * @regs: NAND controller registers - * @ahb_clk: NAND Controller AHB clock - * @mod_clk: NAND Controller mod clock - * @assigned_cs: bitmask describing already assigned CS lines - * @clk_rate: NAND controller current clock rate - * @chips: a list containing all the NAND chips attached to - * this NAND controller - * @complete: a completion object used to wait for NAND - * controller events + * @controller: base controller structure + * @dev: parent device (used to print error messages) + * @regs: NAND controller registers + * @ahb_clk: NAND controller AHB clock + * @mod_clk: NAND controller mod clock + * @reset: NAND controller reset line + * @assigned_cs: bitmask describing already assigned CS lines + * @clk_rate: NAND controller current clock rate + * @chips: a list containing all the NAND chips attached to this NAND + * controller + * @complete: a completion object used to wait for NAND controller events + * @dmac: the DMA channel attached to the NAND controller */ struct sunxi_nfc { struct nand_controller controller; @@ -339,13 +324,11 @@ static int sunxi_nfc_rst(struct sunxi_nfc *nfc) return ret; } -static int sunxi_nfc_dma_op_prepare(struct mtd_info *mtd, const void *buf, +static int sunxi_nfc_dma_op_prepare(struct sunxi_nfc *nfc, const void *buf, int chunksize, int nchunks, enum dma_data_direction ddir, struct scatterlist *sg) { - struct nand_chip *nand = mtd_to_nand(mtd); - struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); struct dma_async_tx_descriptor *dmad; enum dma_transfer_direction tdir; dma_cookie_t dmat; @@ -388,38 +371,16 @@ err_unmap_buf: return ret; } -static void sunxi_nfc_dma_op_cleanup(struct mtd_info *mtd, +static void sunxi_nfc_dma_op_cleanup(struct sunxi_nfc *nfc, enum dma_data_direction ddir, struct scatterlist *sg) { - struct nand_chip *nand = mtd_to_nand(mtd); - struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); - dma_unmap_sg(nfc->dev, sg, 1, ddir); writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD, nfc->regs + NFC_REG_CTL); } -static int sunxi_nfc_dev_ready(struct nand_chip *nand) -{ - struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); - struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); - u32 mask; - - if (sunxi_nand->selected < 0) - return 0; - - if (sunxi_nand->sels[sunxi_nand->selected].rb < 0) { - dev_err(nfc->dev, "cannot check R/B NAND status!\n"); - return 0; - } - - mask = NFC_RB_STATE(sunxi_nand->sels[sunxi_nand->selected].rb); - - return !!(readl(nfc->regs + NFC_REG_ST) & mask); -} - -static void sunxi_nfc_select_chip(struct nand_chip *nand, int chip) +static void sunxi_nfc_select_chip(struct nand_chip *nand, unsigned int cs) { struct mtd_info *mtd = nand_to_mtd(nand); struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); @@ -427,40 +388,27 @@ static void sunxi_nfc_select_chip(struct nand_chip *nand, int chip) struct sunxi_nand_chip_sel *sel; u32 ctl; - if (chip > 0 && chip >= sunxi_nand->nsels) - return; - - if (chip == sunxi_nand->selected) + if (cs > 0 && cs >= sunxi_nand->nsels) return; ctl = readl(nfc->regs + NFC_REG_CTL) & ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN); - if (chip >= 0) { - sel = &sunxi_nand->sels[chip]; + sel = &sunxi_nand->sels[cs]; + ctl |= NFC_CE_SEL(sel->cs) | NFC_EN | NFC_PAGE_SHIFT(nand->page_shift); + if (sel->rb >= 0) + ctl |= NFC_RB_SEL(sel->rb); - ctl |= NFC_CE_SEL(sel->cs) | NFC_EN | - NFC_PAGE_SHIFT(nand->page_shift); - if (sel->rb < 0) { - nand->legacy.dev_ready = NULL; - } else { - nand->legacy.dev_ready = sunxi_nfc_dev_ready; - ctl |= NFC_RB_SEL(sel->rb); - } - - writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA); + writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA); - if (nfc->clk_rate != sunxi_nand->clk_rate) { - clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate); - nfc->clk_rate = sunxi_nand->clk_rate; - } + if (nfc->clk_rate != sunxi_nand->clk_rate) { + clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate); + nfc->clk_rate = sunxi_nand->clk_rate; } writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL); writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG); writel(ctl, nfc->regs + NFC_REG_CTL); - - sunxi_nand->selected = chip; } static void sunxi_nfc_read_buf(struct nand_chip *nand, uint8_t *buf, int len) @@ -537,71 +485,6 @@ static void sunxi_nfc_write_buf(struct nand_chip *nand, const uint8_t *buf, } } -static uint8_t sunxi_nfc_read_byte(struct nand_chip *nand) -{ - uint8_t ret = 0; - - sunxi_nfc_read_buf(nand, &ret, 1); - - return ret; -} - -static void sunxi_nfc_cmd_ctrl(struct nand_chip *nand, int dat, - unsigned int ctrl) -{ - struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); - struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); - int ret; - - if (dat == NAND_CMD_NONE && (ctrl & NAND_NCE) && - !(ctrl & (NAND_CLE | NAND_ALE))) { - u32 cmd = 0; - - if (!sunxi_nand->addr_cycles && !sunxi_nand->cmd_cycles) - return; - - if (sunxi_nand->cmd_cycles--) - cmd |= NFC_SEND_CMD1 | sunxi_nand->cmd[0]; - - if (sunxi_nand->cmd_cycles--) { - cmd |= NFC_SEND_CMD2; - writel(sunxi_nand->cmd[1], - nfc->regs + NFC_REG_RCMD_SET); - } - - sunxi_nand->cmd_cycles = 0; - - if (sunxi_nand->addr_cycles) { - cmd |= NFC_SEND_ADR | - NFC_ADR_NUM(sunxi_nand->addr_cycles); - writel(sunxi_nand->addr[0], - nfc->regs + NFC_REG_ADDR_LOW); - } - - if (sunxi_nand->addr_cycles > 4) - writel(sunxi_nand->addr[1], - nfc->regs + NFC_REG_ADDR_HIGH); - - ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); - if (ret) - return; - - writel(cmd, nfc->regs + NFC_REG_CMD); - sunxi_nand->addr[0] = 0; - sunxi_nand->addr[1] = 0; - sunxi_nand->addr_cycles = 0; - sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0); - } - - if (ctrl & NAND_CLE) { - sunxi_nand->cmd[sunxi_nand->cmd_cycles++] = dat; - } else if (ctrl & NAND_ALE) { - sunxi_nand->addr[sunxi_nand->addr_cycles / 4] |= - dat << ((sunxi_nand->addr_cycles % 4) * 8); - sunxi_nand->addr_cycles++; - } -} - /* These seed values have been extracted from Allwinner's BSP */ static const u16 sunxi_nfc_randomizer_page_seeds[] = { 0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72, @@ -684,8 +567,10 @@ static u16 sunxi_nfc_randomizer_step(u16 state, int count) return state; } -static u16 sunxi_nfc_randomizer_state(struct mtd_info *mtd, int page, bool ecc) +static u16 sunxi_nfc_randomizer_state(struct nand_chip *nand, int page, + bool ecc) { + struct mtd_info *mtd = nand_to_mtd(nand); const u16 *seeds = sunxi_nfc_randomizer_page_seeds; int mod = mtd_div_by_ws(mtd->erasesize, mtd); @@ -702,10 +587,9 @@ static u16 sunxi_nfc_randomizer_state(struct mtd_info *mtd, int page, bool ecc) return seeds[page % mod]; } -static void sunxi_nfc_randomizer_config(struct mtd_info *mtd, - int page, bool ecc) +static void sunxi_nfc_randomizer_config(struct nand_chip *nand, int page, + bool ecc) { - struct nand_chip *nand = mtd_to_nand(mtd); struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL); u16 state; @@ -714,14 +598,13 @@ static void sunxi_nfc_randomizer_config(struct mtd_info *mtd, return; ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL); - state = sunxi_nfc_randomizer_state(mtd, page, ecc); + state = sunxi_nfc_randomizer_state(nand, page, ecc); ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK; writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL); } -static void sunxi_nfc_randomizer_enable(struct mtd_info *mtd) +static void sunxi_nfc_randomizer_enable(struct nand_chip *nand) { - struct nand_chip *nand = mtd_to_nand(mtd); struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); if (!(nand->options & NAND_NEED_SCRAMBLING)) @@ -731,9 +614,8 @@ static void sunxi_nfc_randomizer_enable(struct mtd_info *mtd) nfc->regs + NFC_REG_ECC_CTL); } -static void sunxi_nfc_randomizer_disable(struct mtd_info *mtd) +static void sunxi_nfc_randomizer_disable(struct nand_chip *nand) { - struct nand_chip *nand = mtd_to_nand(mtd); struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); if (!(nand->options & NAND_NEED_SCRAMBLING)) @@ -743,36 +625,35 @@ static void sunxi_nfc_randomizer_disable(struct mtd_info *mtd) nfc->regs + NFC_REG_ECC_CTL); } -static void sunxi_nfc_randomize_bbm(struct mtd_info *mtd, int page, u8 *bbm) +static void sunxi_nfc_randomize_bbm(struct nand_chip *nand, int page, u8 *bbm) { - u16 state = sunxi_nfc_randomizer_state(mtd, page, true); + u16 state = sunxi_nfc_randomizer_state(nand, page, true); bbm[0] ^= state; bbm[1] ^= sunxi_nfc_randomizer_step(state, 8); } -static void sunxi_nfc_randomizer_write_buf(struct mtd_info *mtd, +static void sunxi_nfc_randomizer_write_buf(struct nand_chip *nand, const uint8_t *buf, int len, bool ecc, int page) { - sunxi_nfc_randomizer_config(mtd, page, ecc); - sunxi_nfc_randomizer_enable(mtd); - sunxi_nfc_write_buf(mtd_to_nand(mtd), buf, len); - sunxi_nfc_randomizer_disable(mtd); + sunxi_nfc_randomizer_config(nand, page, ecc); + sunxi_nfc_randomizer_enable(nand); + sunxi_nfc_write_buf(nand, buf, len); + sunxi_nfc_randomizer_disable(nand); } -static void sunxi_nfc_randomizer_read_buf(struct mtd_info *mtd, uint8_t *buf, +static void sunxi_nfc_randomizer_read_buf(struct nand_chip *nand, uint8_t *buf, int len, bool ecc, int page) { - sunxi_nfc_randomizer_config(mtd, page, ecc); - sunxi_nfc_randomizer_enable(mtd); - sunxi_nfc_read_buf(mtd_to_nand(mtd), buf, len); - sunxi_nfc_randomizer_disable(mtd); + sunxi_nfc_randomizer_config(nand, page, ecc); + sunxi_nfc_randomizer_enable(nand); + sunxi_nfc_read_buf(nand, buf, len); + sunxi_nfc_randomizer_disable(nand); } -static void sunxi_nfc_hw_ecc_enable(struct mtd_info *mtd) +static void sunxi_nfc_hw_ecc_enable(struct nand_chip *nand) { - struct nand_chip *nand = mtd_to_nand(mtd); struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); struct sunxi_nand_hw_ecc *data = nand->ecc.priv; u32 ecc_ctl; @@ -789,9 +670,8 @@ static void sunxi_nfc_hw_ecc_enable(struct mtd_info *mtd) writel(ecc_ctl, nfc->regs + NFC_REG_ECC_CTL); } -static void sunxi_nfc_hw_ecc_disable(struct mtd_info *mtd) +static void sunxi_nfc_hw_ecc_disable(struct nand_chip *nand) { - struct nand_chip *nand = mtd_to_nand(mtd); struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_ECC_EN, @@ -811,10 +691,9 @@ static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf) return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24); } -static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct mtd_info *mtd, u8 *oob, +static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct nand_chip *nand, u8 *oob, int step, bool bbm, int page) { - struct nand_chip *nand = mtd_to_nand(mtd); struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)), @@ -822,21 +701,20 @@ static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct mtd_info *mtd, u8 *oob, /* De-randomize the Bad Block Marker. */ if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) - sunxi_nfc_randomize_bbm(mtd, page, oob); + sunxi_nfc_randomize_bbm(nand, page, oob); } -static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct mtd_info *mtd, +static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct nand_chip *nand, const u8 *oob, int step, bool bbm, int page) { - struct nand_chip *nand = mtd_to_nand(mtd); struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); u8 user_data[4]; /* Randomize the Bad Block Marker. */ if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) { memcpy(user_data, oob, sizeof(user_data)); - sunxi_nfc_randomize_bbm(mtd, page, user_data); + sunxi_nfc_randomize_bbm(nand, page, user_data); oob = user_data; } @@ -844,9 +722,11 @@ static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct mtd_info *mtd, nfc->regs + NFC_REG_USER_DATA(step)); } -static void sunxi_nfc_hw_ecc_update_stats(struct mtd_info *mtd, +static void sunxi_nfc_hw_ecc_update_stats(struct nand_chip *nand, unsigned int *max_bitflips, int ret) { + struct mtd_info *mtd = nand_to_mtd(nand); + if (ret < 0) { mtd->ecc_stats.failed++; } else { @@ -855,10 +735,9 @@ static void sunxi_nfc_hw_ecc_update_stats(struct mtd_info *mtd, } } -static int sunxi_nfc_hw_ecc_correct(struct mtd_info *mtd, u8 *data, u8 *oob, +static int sunxi_nfc_hw_ecc_correct(struct nand_chip *nand, u8 *data, u8 *oob, int step, u32 status, bool *erased) { - struct nand_chip *nand = mtd_to_nand(mtd); struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); struct nand_ecc_ctrl *ecc = &nand->ecc; u32 tmp; @@ -892,14 +771,13 @@ static int sunxi_nfc_hw_ecc_correct(struct mtd_info *mtd, u8 *data, u8 *oob, return NFC_ECC_ERR_CNT(step, tmp); } -static int sunxi_nfc_hw_ecc_read_chunk(struct mtd_info *mtd, +static int sunxi_nfc_hw_ecc_read_chunk(struct nand_chip *nand, u8 *data, int data_off, u8 *oob, int oob_off, int *cur_off, unsigned int *max_bitflips, bool bbm, bool oob_required, int page) { - struct nand_chip *nand = mtd_to_nand(mtd); struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); struct nand_ecc_ctrl *ecc = &nand->ecc; int raw_mode = 0; @@ -909,7 +787,7 @@ static int sunxi_nfc_hw_ecc_read_chunk(struct mtd_info *mtd, if (*cur_off != data_off) nand_change_read_column_op(nand, data_off, NULL, 0, false); - sunxi_nfc_randomizer_read_buf(mtd, NULL, ecc->size, false, page); + sunxi_nfc_randomizer_read_buf(nand, NULL, ecc->size, false, page); if (data_off + ecc->size != oob_off) nand_change_read_column_op(nand, oob_off, NULL, 0, false); @@ -918,18 +796,18 @@ static int sunxi_nfc_hw_ecc_read_chunk(struct mtd_info *mtd, if (ret) return ret; - sunxi_nfc_randomizer_enable(mtd); + sunxi_nfc_randomizer_enable(nand); writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP, nfc->regs + NFC_REG_CMD); ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0); - sunxi_nfc_randomizer_disable(mtd); + sunxi_nfc_randomizer_disable(nand); if (ret) return ret; *cur_off = oob_off + ecc->bytes + 4; - ret = sunxi_nfc_hw_ecc_correct(mtd, data, oob_required ? oob : NULL, 0, + ret = sunxi_nfc_hw_ecc_correct(nand, data, oob_required ? oob : NULL, 0, readl(nfc->regs + NFC_REG_ECC_ST), &erased); if (erased) @@ -961,24 +839,24 @@ static int sunxi_nfc_hw_ecc_read_chunk(struct mtd_info *mtd, if (oob_required) { nand_change_read_column_op(nand, oob_off, NULL, 0, false); - sunxi_nfc_randomizer_read_buf(mtd, oob, ecc->bytes + 4, + sunxi_nfc_randomizer_read_buf(nand, oob, ecc->bytes + 4, true, page); - sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, 0, + sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, 0, bbm, page); } } - sunxi_nfc_hw_ecc_update_stats(mtd, max_bitflips, ret); + sunxi_nfc_hw_ecc_update_stats(nand, max_bitflips, ret); return raw_mode; } -static void sunxi_nfc_hw_ecc_read_extra_oob(struct mtd_info *mtd, +static void sunxi_nfc_hw_ecc_read_extra_oob(struct nand_chip *nand, u8 *oob, int *cur_off, bool randomize, int page) { - struct nand_chip *nand = mtd_to_nand(mtd); + struct mtd_info *mtd = nand_to_mtd(nand); struct nand_ecc_ctrl *ecc = &nand->ecc; int offset = ((ecc->bytes + 4) * ecc->steps); int len = mtd->oobsize - offset; @@ -993,20 +871,20 @@ static void sunxi_nfc_hw_ecc_read_extra_oob(struct mtd_info *mtd, if (!randomize) sunxi_nfc_read_buf(nand, oob + offset, len); else - sunxi_nfc_randomizer_read_buf(mtd, oob + offset, len, + sunxi_nfc_randomizer_read_buf(nand, oob + offset, len, false, page); if (cur_off) *cur_off = mtd->oobsize + mtd->writesize; } -static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf, +static int sunxi_nfc_hw_ecc_read_chunks_dma(struct nand_chip *nand, uint8_t *buf, int oob_required, int page, int nchunks) { - struct nand_chip *nand = mtd_to_nand(mtd); bool randomized = nand->options & NAND_NEED_SCRAMBLING; struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); + struct mtd_info *mtd = nand_to_mtd(nand); struct nand_ecc_ctrl *ecc = &nand->ecc; unsigned int max_bitflips = 0; int ret, i, raw_mode = 0; @@ -1017,14 +895,14 @@ static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf, if (ret) return ret; - ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, nchunks, + ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, nchunks, DMA_FROM_DEVICE, &sg); if (ret) return ret; - sunxi_nfc_hw_ecc_enable(mtd); - sunxi_nfc_randomizer_config(mtd, page, false); - sunxi_nfc_randomizer_enable(mtd); + sunxi_nfc_hw_ecc_enable(nand); + sunxi_nfc_randomizer_config(nand, page, false); + sunxi_nfc_randomizer_enable(nand); writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) | NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET); @@ -1038,10 +916,10 @@ static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf, if (ret) dmaengine_terminate_all(nfc->dmac); - sunxi_nfc_randomizer_disable(mtd); - sunxi_nfc_hw_ecc_disable(mtd); + sunxi_nfc_randomizer_disable(nand); + sunxi_nfc_hw_ecc_disable(nand); - sunxi_nfc_dma_op_cleanup(mtd, DMA_FROM_DEVICE, &sg); + sunxi_nfc_dma_op_cleanup(nfc, DMA_FROM_DEVICE, &sg); if (ret) return ret; @@ -1055,7 +933,7 @@ static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf, u8 *oob = nand->oob_poi + oob_off; bool erased; - ret = sunxi_nfc_hw_ecc_correct(mtd, randomized ? data : NULL, + ret = sunxi_nfc_hw_ecc_correct(nand, randomized ? data : NULL, oob_required ? oob : NULL, i, status, &erased); @@ -1069,14 +947,14 @@ static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf, mtd->writesize + oob_off, oob, ecc->bytes + 4, false); - sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, i, + sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, i, !i, page); } if (erased) raw_mode = 1; - sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret); + sunxi_nfc_hw_ecc_update_stats(nand, &max_bitflips, ret); } if (status & NFC_ECC_ERR_MSK) { @@ -1111,25 +989,24 @@ static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf, if (ret >= 0) raw_mode = 1; - sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret); + sunxi_nfc_hw_ecc_update_stats(nand, &max_bitflips, ret); } } if (oob_required) - sunxi_nfc_hw_ecc_read_extra_oob(mtd, nand->oob_poi, + sunxi_nfc_hw_ecc_read_extra_oob(nand, nand->oob_poi, NULL, !raw_mode, page); return max_bitflips; } -static int sunxi_nfc_hw_ecc_write_chunk(struct mtd_info *mtd, +static int sunxi_nfc_hw_ecc_write_chunk(struct nand_chip *nand, const u8 *data, int data_off, const u8 *oob, int oob_off, int *cur_off, bool bbm, int page) { - struct nand_chip *nand = mtd_to_nand(mtd); struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); struct nand_ecc_ctrl *ecc = &nand->ecc; int ret; @@ -1137,7 +1014,7 @@ static int sunxi_nfc_hw_ecc_write_chunk(struct mtd_info *mtd, if (data_off != *cur_off) nand_change_write_column_op(nand, data_off, NULL, 0, false); - sunxi_nfc_randomizer_write_buf(mtd, data, ecc->size, false, page); + sunxi_nfc_randomizer_write_buf(nand, data, ecc->size, false, page); if (data_off + ecc->size != oob_off) nand_change_write_column_op(nand, oob_off, NULL, 0, false); @@ -1146,15 +1023,15 @@ static int sunxi_nfc_hw_ecc_write_chunk(struct mtd_info *mtd, if (ret) return ret; - sunxi_nfc_randomizer_enable(mtd); - sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, 0, bbm, page); + sunxi_nfc_randomizer_enable(nand); + sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, 0, bbm, page); writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ACCESS_DIR | NFC_ECC_OP, nfc->regs + NFC_REG_CMD); ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0); - sunxi_nfc_randomizer_disable(mtd); + sunxi_nfc_randomizer_disable(nand); if (ret) return ret; @@ -1163,11 +1040,11 @@ static int sunxi_nfc_hw_ecc_write_chunk(struct mtd_info *mtd, return 0; } -static void sunxi_nfc_hw_ecc_write_extra_oob(struct mtd_info *mtd, +static void sunxi_nfc_hw_ecc_write_extra_oob(struct nand_chip *nand, u8 *oob, int *cur_off, int page) { - struct nand_chip *nand = mtd_to_nand(mtd); + struct mtd_info *mtd = nand_to_mtd(nand); struct nand_ecc_ctrl *ecc = &nand->ecc; int offset = ((ecc->bytes + 4) * ecc->steps); int len = mtd->oobsize - offset; @@ -1179,32 +1056,34 @@ static void sunxi_nfc_hw_ecc_write_extra_oob(struct mtd_info *mtd, nand_change_write_column_op(nand, offset + mtd->writesize, NULL, 0, false); - sunxi_nfc_randomizer_write_buf(mtd, oob + offset, len, false, page); + sunxi_nfc_randomizer_write_buf(nand, oob + offset, len, false, page); if (cur_off) *cur_off = mtd->oobsize + mtd->writesize; } -static int sunxi_nfc_hw_ecc_read_page(struct nand_chip *chip, uint8_t *buf, +static int sunxi_nfc_hw_ecc_read_page(struct nand_chip *nand, uint8_t *buf, int oob_required, int page) { - struct mtd_info *mtd = nand_to_mtd(chip); - struct nand_ecc_ctrl *ecc = &chip->ecc; + struct mtd_info *mtd = nand_to_mtd(nand); + struct nand_ecc_ctrl *ecc = &nand->ecc; unsigned int max_bitflips = 0; int ret, i, cur_off = 0; bool raw_mode = false; - nand_read_page_op(chip, page, 0, NULL, 0); + sunxi_nfc_select_chip(nand, nand->cur_cs); + + nand_read_page_op(nand, page, 0, NULL, 0); - sunxi_nfc_hw_ecc_enable(mtd); + sunxi_nfc_hw_ecc_enable(nand); for (i = 0; i < ecc->steps; i++) { int data_off = i * ecc->size; int oob_off = i * (ecc->bytes + 4); u8 *data = buf + data_off; - u8 *oob = chip->oob_poi + oob_off; + u8 *oob = nand->oob_poi + oob_off; - ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob, + ret = sunxi_nfc_hw_ecc_read_chunk(nand, data, data_off, oob, oob_off + mtd->writesize, &cur_off, &max_bitflips, !i, oob_required, page); @@ -1215,52 +1094,55 @@ static int sunxi_nfc_hw_ecc_read_page(struct nand_chip *chip, uint8_t *buf, } if (oob_required) - sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off, + sunxi_nfc_hw_ecc_read_extra_oob(nand, nand->oob_poi, &cur_off, !raw_mode, page); - sunxi_nfc_hw_ecc_disable(mtd); + sunxi_nfc_hw_ecc_disable(nand); return max_bitflips; } -static int sunxi_nfc_hw_ecc_read_page_dma(struct nand_chip *chip, u8 *buf, +static int sunxi_nfc_hw_ecc_read_page_dma(struct nand_chip *nand, u8 *buf, int oob_required, int page) { - struct mtd_info *mtd = nand_to_mtd(chip); int ret; - nand_read_page_op(chip, page, 0, NULL, 0); + sunxi_nfc_select_chip(nand, nand->cur_cs); + + nand_read_page_op(nand, page, 0, NULL, 0); - ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, oob_required, page, - chip->ecc.steps); + ret = sunxi_nfc_hw_ecc_read_chunks_dma(nand, buf, oob_required, page, + nand->ecc.steps); if (ret >= 0) return ret; /* Fallback to PIO mode */ - return sunxi_nfc_hw_ecc_read_page(chip, buf, oob_required, page); + return sunxi_nfc_hw_ecc_read_page(nand, buf, oob_required, page); } -static int sunxi_nfc_hw_ecc_read_subpage(struct nand_chip *chip, +static int sunxi_nfc_hw_ecc_read_subpage(struct nand_chip *nand, u32 data_offs, u32 readlen, u8 *bufpoi, int page) { - struct mtd_info *mtd = nand_to_mtd(chip); - struct nand_ecc_ctrl *ecc = &chip->ecc; + struct mtd_info *mtd = nand_to_mtd(nand); + struct nand_ecc_ctrl *ecc = &nand->ecc; int ret, i, cur_off = 0; unsigned int max_bitflips = 0; - nand_read_page_op(chip, page, 0, NULL, 0); + sunxi_nfc_select_chip(nand, nand->cur_cs); + + nand_read_page_op(nand, page, 0, NULL, 0); - sunxi_nfc_hw_ecc_enable(mtd); + sunxi_nfc_hw_ecc_enable(nand); for (i = data_offs / ecc->size; i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) { int data_off = i * ecc->size; int oob_off = i * (ecc->bytes + 4); u8 *data = bufpoi + data_off; - u8 *oob = chip->oob_poi + oob_off; + u8 *oob = nand->oob_poi + oob_off; - ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, + ret = sunxi_nfc_hw_ecc_read_chunk(nand, data, data_off, oob, oob_off + mtd->writesize, &cur_off, &max_bitflips, !i, @@ -1269,113 +1151,118 @@ static int sunxi_nfc_hw_ecc_read_subpage(struct nand_chip *chip, return ret; } - sunxi_nfc_hw_ecc_disable(mtd); + sunxi_nfc_hw_ecc_disable(nand); return max_bitflips; } -static int sunxi_nfc_hw_ecc_read_subpage_dma(struct nand_chip *chip, +static int sunxi_nfc_hw_ecc_read_subpage_dma(struct nand_chip *nand, u32 data_offs, u32 readlen, u8 *buf, int page) { - struct mtd_info *mtd = nand_to_mtd(chip); - int nchunks = DIV_ROUND_UP(data_offs + readlen, chip->ecc.size); + int nchunks = DIV_ROUND_UP(data_offs + readlen, nand->ecc.size); int ret; - nand_read_page_op(chip, page, 0, NULL, 0); + sunxi_nfc_select_chip(nand, nand->cur_cs); + + nand_read_page_op(nand, page, 0, NULL, 0); - ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, false, page, nchunks); + ret = sunxi_nfc_hw_ecc_read_chunks_dma(nand, buf, false, page, nchunks); if (ret >= 0) return ret; /* Fallback to PIO mode */ - return sunxi_nfc_hw_ecc_read_subpage(chip, data_offs, readlen, + return sunxi_nfc_hw_ecc_read_subpage(nand, data_offs, readlen, buf, page); } -static int sunxi_nfc_hw_ecc_write_page(struct nand_chip *chip, +static int sunxi_nfc_hw_ecc_write_page(struct nand_chip *nand, const uint8_t *buf, int oob_required, int page) { - struct mtd_info *mtd = nand_to_mtd(chip); - struct nand_ecc_ctrl *ecc = &chip->ecc; + struct mtd_info *mtd = nand_to_mtd(nand); + struct nand_ecc_ctrl *ecc = &nand->ecc; int ret, i, cur_off = 0; - nand_prog_page_begin_op(chip, page, 0, NULL, 0); + sunxi_nfc_select_chip(nand, nand->cur_cs); + + nand_prog_page_begin_op(nand, page, 0, NULL, 0); - sunxi_nfc_hw_ecc_enable(mtd); + sunxi_nfc_hw_ecc_enable(nand); for (i = 0; i < ecc->steps; i++) { int data_off = i * ecc->size; int oob_off = i * (ecc->bytes + 4); const u8 *data = buf + data_off; - const u8 *oob = chip->oob_poi + oob_off; + const u8 *oob = nand->oob_poi + oob_off; - ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob, + ret = sunxi_nfc_hw_ecc_write_chunk(nand, data, data_off, oob, oob_off + mtd->writesize, &cur_off, !i, page); if (ret) return ret; } - if (oob_required || (chip->options & NAND_NEED_SCRAMBLING)) - sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi, + if (oob_required || (nand->options & NAND_NEED_SCRAMBLING)) + sunxi_nfc_hw_ecc_write_extra_oob(nand, nand->oob_poi, &cur_off, page); - sunxi_nfc_hw_ecc_disable(mtd); + sunxi_nfc_hw_ecc_disable(nand); - return nand_prog_page_end_op(chip); + return nand_prog_page_end_op(nand); } -static int sunxi_nfc_hw_ecc_write_subpage(struct nand_chip *chip, +static int sunxi_nfc_hw_ecc_write_subpage(struct nand_chip *nand, u32 data_offs, u32 data_len, const u8 *buf, int oob_required, int page) { - struct mtd_info *mtd = nand_to_mtd(chip); - struct nand_ecc_ctrl *ecc = &chip->ecc; + struct mtd_info *mtd = nand_to_mtd(nand); + struct nand_ecc_ctrl *ecc = &nand->ecc; int ret, i, cur_off = 0; - nand_prog_page_begin_op(chip, page, 0, NULL, 0); + sunxi_nfc_select_chip(nand, nand->cur_cs); - sunxi_nfc_hw_ecc_enable(mtd); + nand_prog_page_begin_op(nand, page, 0, NULL, 0); + + sunxi_nfc_hw_ecc_enable(nand); for (i = data_offs / ecc->size; i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) { int data_off = i * ecc->size; int oob_off = i * (ecc->bytes + 4); const u8 *data = buf + data_off; - const u8 *oob = chip->oob_poi + oob_off; + const u8 *oob = nand->oob_poi + oob_off; - ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob, + ret = sunxi_nfc_hw_ecc_write_chunk(nand, data, data_off, oob, oob_off + mtd->writesize, &cur_off, !i, page); if (ret) return ret; } - sunxi_nfc_hw_ecc_disable(mtd); + sunxi_nfc_hw_ecc_disable(nand); - return nand_prog_page_end_op(chip); + return nand_prog_page_end_op(nand); } -static int sunxi_nfc_hw_ecc_write_page_dma(struct nand_chip *chip, +static int sunxi_nfc_hw_ecc_write_page_dma(struct nand_chip *nand, const u8 *buf, int oob_required, int page) { - struct mtd_info *mtd = nand_to_mtd(chip); - struct nand_chip *nand = mtd_to_nand(mtd); struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); struct nand_ecc_ctrl *ecc = &nand->ecc; struct scatterlist sg; int ret, i; + sunxi_nfc_select_chip(nand, nand->cur_cs); + ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); if (ret) return ret; - ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, ecc->steps, + ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, ecc->steps, DMA_TO_DEVICE, &sg); if (ret) goto pio_fallback; @@ -1383,14 +1270,14 @@ static int sunxi_nfc_hw_ecc_write_page_dma(struct nand_chip *chip, for (i = 0; i < ecc->steps; i++) { const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4)); - sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, i, !i, page); + sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, i, !i, page); } - nand_prog_page_begin_op(chip, page, 0, NULL, 0); + nand_prog_page_begin_op(nand, page, 0, NULL, 0); - sunxi_nfc_hw_ecc_enable(mtd); - sunxi_nfc_randomizer_config(mtd, page, false); - sunxi_nfc_randomizer_enable(mtd); + sunxi_nfc_hw_ecc_enable(nand); + sunxi_nfc_randomizer_config(nand, page, false); + sunxi_nfc_randomizer_enable(nand); writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG, nfc->regs + NFC_REG_WCMD_SET); @@ -1405,46 +1292,46 @@ static int sunxi_nfc_hw_ecc_write_page_dma(struct nand_chip *chip, if (ret) dmaengine_terminate_all(nfc->dmac); - sunxi_nfc_randomizer_disable(mtd); - sunxi_nfc_hw_ecc_disable(mtd); + sunxi_nfc_randomizer_disable(nand); + sunxi_nfc_hw_ecc_disable(nand); - sunxi_nfc_dma_op_cleanup(mtd, DMA_TO_DEVICE, &sg); + sunxi_nfc_dma_op_cleanup(nfc, DMA_TO_DEVICE, &sg); if (ret) return ret; - if (oob_required || (chip->options & NAND_NEED_SCRAMBLING)) + if (oob_required || (nand->options & NAND_NEED_SCRAMBLING)) /* TODO: use DMA to transfer extra OOB bytes ? */ - sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi, + sunxi_nfc_hw_ecc_write_extra_oob(nand, nand->oob_poi, NULL, page); - return nand_prog_page_end_op(chip); + return nand_prog_page_end_op(nand); pio_fallback: - return sunxi_nfc_hw_ecc_write_page(chip, buf, oob_required, page); + return sunxi_nfc_hw_ecc_write_page(nand, buf, oob_required, page); } -static int sunxi_nfc_hw_ecc_read_oob(struct nand_chip *chip, int page) +static int sunxi_nfc_hw_ecc_read_oob(struct nand_chip *nand, int page) { - chip->pagebuf = -1; + nand->pagebuf = -1; - return chip->ecc.read_page(chip, chip->data_buf, 1, page); + return nand->ecc.read_page(nand, nand->data_buf, 1, page); } -static int sunxi_nfc_hw_ecc_write_oob(struct nand_chip *chip, int page) +static int sunxi_nfc_hw_ecc_write_oob(struct nand_chip *nand, int page) { - struct mtd_info *mtd = nand_to_mtd(chip); + struct mtd_info *mtd = nand_to_mtd(nand); int ret; - chip->pagebuf = -1; + nand->pagebuf = -1; - memset(chip->data_buf, 0xff, mtd->writesize); - ret = chip->ecc.write_page(chip, chip->data_buf, 1, page); + memset(nand->data_buf, 0xff, mtd->writesize); + ret = nand->ecc.write_page(nand, nand->data_buf, 1, page); if (ret) return ret; /* Send command to program the OOB data */ - return nand_prog_page_end_op(chip); + return nand_prog_page_end_op(nand); } static const s32 tWB_lut[] = {6, 12, 16, 20}; @@ -1471,8 +1358,8 @@ static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration, static int sunxi_nfc_setup_data_interface(struct nand_chip *nand, int csline, const struct nand_data_interface *conf) { - struct sunxi_nand_chip *chip = to_sunxi_nand(nand); - struct sunxi_nfc *nfc = to_sunxi_nfc(chip->nand.controller); + struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); + struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); const struct nand_sdr_timings *timings; u32 min_clk_period = 0; s32 tWB, tADL, tWHR, tRHW, tCAD; @@ -1555,6 +1442,20 @@ static int sunxi_nfc_setup_data_interface(struct nand_chip *nand, int csline, if (timings->tRHW_min > (min_clk_period * 20)) min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20); + /* + * In non-EDO, tREA should be less than tRP to guarantee that the + * controller does not sample the IO lines too early. Unfortunately, + * the sunxi NAND controller does not allow us to have different + * values for tRP and tREH (tRP = tREH = tRW / 2). + * + * We have 2 options to overcome this limitation: + * + * 1/ Extend tRC to fulfil the tREA <= tRC / 2 constraint + * 2/ Use EDO mode (only works if timings->tRLOH > 0) + */ + if (timings->tREA_max > min_clk_period && !timings->tRLOH_min) + min_clk_period = timings->tREA_max; + tWB = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max, min_clk_period); if (tWB < 0) { @@ -1591,7 +1492,7 @@ static int sunxi_nfc_setup_data_interface(struct nand_chip *nand, int csline, tCAD = 0x7; /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */ - chip->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD); + sunxi_nand->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD); /* Convert min_clk_period from picoseconds to nanoseconds */ min_clk_period = DIV_ROUND_UP(min_clk_period, 1000); @@ -1602,21 +1503,24 @@ static int sunxi_nfc_setup_data_interface(struct nand_chip *nand, int csline, * This new formula was verified with a scope and validated by * Allwinner engineers. */ - chip->clk_rate = NSEC_PER_SEC / min_clk_period; - real_clk_rate = clk_round_rate(nfc->mod_clk, chip->clk_rate); + sunxi_nand->clk_rate = NSEC_PER_SEC / min_clk_period; + real_clk_rate = clk_round_rate(nfc->mod_clk, sunxi_nand->clk_rate); if (real_clk_rate <= 0) { - dev_err(nfc->dev, "Unable to round clk %lu\n", chip->clk_rate); + dev_err(nfc->dev, "Unable to round clk %lu\n", + sunxi_nand->clk_rate); return -EINVAL; } + sunxi_nand->timing_ctl = 0; + /* * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data * output cycle timings shall be used if the host drives tRC less than - * 30 ns. + * 30 ns. We should also use EDO mode if tREA is bigger than tRP. */ min_clk_period = NSEC_PER_SEC / real_clk_rate; - chip->timing_ctl = ((min_clk_period * 2) < 30) ? - NFC_TIMING_CTL_EDO : 0; + if (min_clk_period * 2 < 30 || min_clk_period * 1000 < timings->tREA_max) + sunxi_nand->timing_ctl = NFC_TIMING_CTL_EDO; return 0; } @@ -1677,14 +1581,13 @@ static void sunxi_nand_hw_ecc_ctrl_cleanup(struct nand_ecc_ctrl *ecc) kfree(ecc->priv); } -static int sunxi_nand_hw_ecc_ctrl_init(struct mtd_info *mtd, +static int sunxi_nand_hw_ecc_ctrl_init(struct nand_chip *nand, struct nand_ecc_ctrl *ecc, struct device_node *np) { static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 }; - struct nand_chip *nand = mtd_to_nand(mtd); - struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); - struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller); + struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); + struct mtd_info *mtd = nand_to_mtd(nand); struct sunxi_nand_hw_ecc *data; int nsectors; int ret; @@ -1808,7 +1711,6 @@ static void sunxi_nand_ecc_cleanup(struct nand_ecc_ctrl *ecc) static int sunxi_nand_attach_chip(struct nand_chip *nand) { - struct mtd_info *mtd = nand_to_mtd(nand); struct nand_ecc_ctrl *ecc = &nand->ecc; struct device_node *np = nand_get_flash_node(nand); int ret; @@ -1831,7 +1733,7 @@ static int sunxi_nand_attach_chip(struct nand_chip *nand) switch (ecc->mode) { case NAND_ECC_HW: - ret = sunxi_nand_hw_ecc_ctrl_init(mtd, ecc, np); + ret = sunxi_nand_hw_ecc_ctrl_init(nand, ecc, np); if (ret) return ret; break; @@ -1845,15 +1747,165 @@ static int sunxi_nand_attach_chip(struct nand_chip *nand) return 0; } +static int sunxi_nfc_exec_subop(struct nand_chip *nand, + const struct nand_subop *subop) +{ + struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller); + u32 cmd = 0, extcmd = 0, cnt = 0, addrs[2] = { }; + unsigned int i, j, remaining, start; + void *inbuf = NULL; + int ret; + + for (i = 0; i < subop->ninstrs; i++) { + const struct nand_op_instr *instr = &subop->instrs[i]; + + switch (instr->type) { + case NAND_OP_CMD_INSTR: + if (cmd & NFC_SEND_CMD1) { + if (WARN_ON(cmd & NFC_SEND_CMD2)) + return -EINVAL; + + cmd |= NFC_SEND_CMD2; + extcmd |= instr->ctx.cmd.opcode; + } else { + cmd |= NFC_SEND_CMD1 | + NFC_CMD(instr->ctx.cmd.opcode); + } + break; + + case NAND_OP_ADDR_INSTR: + remaining = nand_subop_get_num_addr_cyc(subop, i); + start = nand_subop_get_addr_start_off(subop, i); + for (j = 0; j < 8 && j + start < remaining; j++) { + u32 addr = instr->ctx.addr.addrs[j + start]; + + addrs[j / 4] |= addr << (j % 4) * 8; + } + + if (j) + cmd |= NFC_SEND_ADR | NFC_ADR_NUM(j); + + break; + + case NAND_OP_DATA_IN_INSTR: + case NAND_OP_DATA_OUT_INSTR: + start = nand_subop_get_data_start_off(subop, i); + remaining = nand_subop_get_data_len(subop, i); + cnt = min_t(u32, remaining, NFC_SRAM_SIZE); + cmd |= NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD; + + if (instr->type == NAND_OP_DATA_OUT_INSTR) { + cmd |= NFC_ACCESS_DIR; + memcpy_toio(nfc->regs + NFC_RAM0_BASE, + instr->ctx.data.buf.out + start, + cnt); + } else { + inbuf = instr->ctx.data.buf.in + start; + } + + break; + + case NAND_OP_WAITRDY_INSTR: + cmd |= NFC_WAIT_FLAG; + break; + } + } + + ret = sunxi_nfc_wait_cmd_fifo_empty(nfc); + if (ret) + return ret; + + if (cmd & NFC_SEND_ADR) { + writel(addrs[0], nfc->regs + NFC_REG_ADDR_LOW); + writel(addrs[1], nfc->regs + NFC_REG_ADDR_HIGH); + } + + if (cmd & NFC_SEND_CMD2) + writel(extcmd, + nfc->regs + + (cmd & NFC_ACCESS_DIR ? + NFC_REG_WCMD_SET : NFC_REG_RCMD_SET)); + + if (cmd & NFC_DATA_TRANS) + writel(cnt, nfc->regs + NFC_REG_CNT); + + writel(cmd, nfc->regs + NFC_REG_CMD); + + ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, + !(cmd & NFC_WAIT_FLAG) && cnt < 64, + 0); + if (ret) + return ret; + + if (inbuf) + memcpy_fromio(inbuf, nfc->regs + NFC_RAM0_BASE, cnt); + + return 0; +} + +static int sunxi_nfc_soft_waitrdy(struct nand_chip *nand, + const struct nand_subop *subop) +{ + return nand_soft_waitrdy(nand, + subop->instrs[0].ctx.waitrdy.timeout_ms); +} + +static const struct nand_op_parser sunxi_nfc_op_parser = NAND_OP_PARSER( + NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop, + NAND_OP_PARSER_PAT_CMD_ELEM(true), + NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8), + NAND_OP_PARSER_PAT_CMD_ELEM(true), + NAND_OP_PARSER_PAT_WAITRDY_ELEM(true), + NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, 1024)), + NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop, + NAND_OP_PARSER_PAT_CMD_ELEM(true), + NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8), + NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, 1024), + NAND_OP_PARSER_PAT_CMD_ELEM(true), + NAND_OP_PARSER_PAT_WAITRDY_ELEM(true)), +); + +static const struct nand_op_parser sunxi_nfc_norb_op_parser = NAND_OP_PARSER( + NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop, + NAND_OP_PARSER_PAT_CMD_ELEM(true), + NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8), + NAND_OP_PARSER_PAT_CMD_ELEM(true), + NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, 1024)), + NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop, + NAND_OP_PARSER_PAT_CMD_ELEM(true), + NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8), + NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, 1024), + NAND_OP_PARSER_PAT_CMD_ELEM(true)), + NAND_OP_PARSER_PATTERN(sunxi_nfc_soft_waitrdy, + NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)), +); + +static int sunxi_nfc_exec_op(struct nand_chip *nand, + const struct nand_operation *op, bool check_only) +{ + struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand); + const struct nand_op_parser *parser; + + sunxi_nfc_select_chip(nand, op->cs); + + if (sunxi_nand->sels[op->cs].rb >= 0) + parser = &sunxi_nfc_op_parser; + else + parser = &sunxi_nfc_norb_op_parser; + + return nand_op_parser_exec_op(nand, parser, op, check_only); +} + static const struct nand_controller_ops sunxi_nand_controller_ops = { .attach_chip = sunxi_nand_attach_chip, .setup_data_interface = sunxi_nfc_setup_data_interface, + .exec_op = sunxi_nfc_exec_op, }; static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc, struct device_node *np) { - struct sunxi_nand_chip *chip; + struct sunxi_nand_chip *sunxi_nand; struct mtd_info *mtd; struct nand_chip *nand; int nsels; @@ -1870,17 +1922,14 @@ static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc, return -EINVAL; } - chip = devm_kzalloc(dev, - sizeof(*chip) + - (nsels * sizeof(struct sunxi_nand_chip_sel)), - GFP_KERNEL); - if (!chip) { + sunxi_nand = devm_kzalloc(dev, struct_size(sunxi_nand, sels, nsels), + GFP_KERNEL); + if (!sunxi_nand) { dev_err(dev, "could not allocate chip\n"); return -ENOMEM; } - chip->nsels = nsels; - chip->selected = -1; + sunxi_nand->nsels = nsels; for (i = 0; i < nsels; i++) { ret = of_property_read_u32_index(np, "reg", i, &tmp); @@ -1902,18 +1951,17 @@ static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc, return -EINVAL; } - chip->sels[i].cs = tmp; + sunxi_nand->sels[i].cs = tmp; if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) && tmp < 2) - chip->sels[i].rb = tmp; + sunxi_nand->sels[i].rb = tmp; else - chip->sels[i].rb = -1; + sunxi_nand->sels[i].rb = -1; } - nand = &chip->nand; + nand = &sunxi_nand->nand; /* Default tR value specified in the ONFI spec (chapter 4.15.1) */ - nand->legacy.chip_delay = 200; nand->controller = &nfc->controller; nand->controller->ops = &sunxi_nand_controller_ops; @@ -1923,11 +1971,6 @@ static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc, */ nand->ecc.mode = NAND_ECC_HW; nand_set_flash_node(nand, np); - nand->legacy.select_chip = sunxi_nfc_select_chip; - nand->legacy.cmd_ctrl = sunxi_nfc_cmd_ctrl; - nand->legacy.read_buf = sunxi_nfc_read_buf; - nand->legacy.write_buf = sunxi_nfc_write_buf; - nand->legacy.read_byte = sunxi_nfc_read_byte; mtd = nand_to_mtd(nand); mtd->dev.parent = dev; @@ -1943,7 +1986,7 @@ static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc, return ret; } - list_add_tail(&chip->node, &nfc->chips); + list_add_tail(&sunxi_nand->node, &nfc->chips); return 0; } @@ -1973,14 +2016,15 @@ static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc) static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc) { - struct sunxi_nand_chip *chip; + struct sunxi_nand_chip *sunxi_nand; while (!list_empty(&nfc->chips)) { - chip = list_first_entry(&nfc->chips, struct sunxi_nand_chip, - node); - nand_release(&chip->nand); - sunxi_nand_ecc_cleanup(&chip->nand.ecc); - list_del(&chip->node); + sunxi_nand = list_first_entry(&nfc->chips, + struct sunxi_nand_chip, + node); + nand_release(&sunxi_nand->nand); + sunxi_nand_ecc_cleanup(&sunxi_nand->nand.ecc); + list_del(&sunxi_nand->node); } } @@ -2124,7 +2168,7 @@ static struct platform_driver sunxi_nfc_driver = { }; module_platform_driver(sunxi_nfc_driver); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); MODULE_AUTHOR("Boris BREZILLON"); MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver"); MODULE_ALIAS("platform:sunxi_nand"); diff --git a/drivers/mtd/nand/raw/tmio_nand.c b/drivers/mtd/nand/raw/tmio_nand.c index f3b59e649b7d..db030f1701ee 100644 --- a/drivers/mtd/nand/raw/tmio_nand.c +++ b/drivers/mtd/nand/raw/tmio_nand.c @@ -104,6 +104,7 @@ struct tmio_nand { struct nand_chip chip; + struct completion comp; struct platform_device *dev; @@ -168,15 +169,11 @@ static int tmio_nand_dev_ready(struct nand_chip *chip) static irqreturn_t tmio_irq(int irq, void *__tmio) { struct tmio_nand *tmio = __tmio; - struct nand_chip *nand_chip = &tmio->chip; /* disable RDYREQ interrupt */ tmio_iowrite8(0x00, tmio->fcr + FCR_IMR); + complete(&tmio->comp); - if (unlikely(!waitqueue_active(&nand_chip->controller->wq))) - dev_warn(&tmio->dev->dev, "spurious interrupt\n"); - - wake_up(&nand_chip->controller->wq); return IRQ_HANDLED; } @@ -193,18 +190,18 @@ static int tmio_nand_wait(struct nand_chip *nand_chip) u8 status; /* enable RDYREQ interrupt */ + tmio_iowrite8(0x0f, tmio->fcr + FCR_ISR); + reinit_completion(&tmio->comp); tmio_iowrite8(0x81, tmio->fcr + FCR_IMR); - timeout = wait_event_timeout(nand_chip->controller->wq, - tmio_nand_dev_ready(nand_chip), - msecs_to_jiffies(nand_chip->state == FL_ERASING ? 400 : 20)); + timeout = 400; + timeout = wait_for_completion_timeout(&tmio->comp, + msecs_to_jiffies(timeout)); if (unlikely(!tmio_nand_dev_ready(nand_chip))) { tmio_iowrite8(0x00, tmio->fcr + FCR_IMR); - dev_warn(&tmio->dev->dev, "still busy with %s after %d ms\n", - nand_chip->state == FL_ERASING ? "erase" : "program", - nand_chip->state == FL_ERASING ? 400 : 20); + dev_warn(&tmio->dev->dev, "still busy after 400 ms\n"); } else if (unlikely(!timeout)) { tmio_iowrite8(0x00, tmio->fcr + FCR_IMR); @@ -378,6 +375,8 @@ static int tmio_probe(struct platform_device *dev) if (!tmio) return -ENOMEM; + init_completion(&tmio->comp); + tmio->dev = dev; platform_set_drvdata(dev, tmio); diff --git a/drivers/mtd/nand/spi/gigadevice.c b/drivers/mtd/nand/spi/gigadevice.c index e4141c20947a..0b49d8264bef 100644 --- a/drivers/mtd/nand/spi/gigadevice.c +++ b/drivers/mtd/nand/spi/gigadevice.c @@ -12,6 +12,8 @@ #define GD5FXGQ4XA_STATUS_ECC_1_7_BITFLIPS (1 << 4) #define GD5FXGQ4XA_STATUS_ECC_8_BITFLIPS (3 << 4) +#define GD5FXGQ4UEXXG_REG_STATUS2 0xf0 + static SPINAND_OP_VARIANTS(read_cache_variants, SPINAND_PAGE_READ_FROM_CACHE_QUADIO_OP(0, 2, NULL, 0), SPINAND_PAGE_READ_FROM_CACHE_X4_OP(0, 1, NULL, 0), @@ -81,11 +83,83 @@ static int gd5fxgq4xa_ecc_get_status(struct spinand_device *spinand, return -EINVAL; } +static int gd5fxgq4uexxg_ooblayout_ecc(struct mtd_info *mtd, int section, + struct mtd_oob_region *region) +{ + if (section) + return -ERANGE; + + region->offset = 64; + region->length = 64; + + return 0; +} + +static int gd5fxgq4uexxg_ooblayout_free(struct mtd_info *mtd, int section, + struct mtd_oob_region *region) +{ + if (section) + return -ERANGE; + + /* Reserve 1 bytes for the BBM. */ + region->offset = 1; + region->length = 63; + + return 0; +} + +static int gd5fxgq4uexxg_ecc_get_status(struct spinand_device *spinand, + u8 status) +{ + u8 status2; + struct spi_mem_op op = SPINAND_GET_FEATURE_OP(GD5FXGQ4UEXXG_REG_STATUS2, + &status2); + int ret; + + switch (status & STATUS_ECC_MASK) { + case STATUS_ECC_NO_BITFLIPS: + return 0; + + case GD5FXGQ4XA_STATUS_ECC_1_7_BITFLIPS: + /* + * Read status2 register to determine a more fine grained + * bit error status + */ + ret = spi_mem_exec_op(spinand->spimem, &op); + if (ret) + return ret; + + /* + * 4 ... 7 bits are flipped (1..4 can't be detected, so + * report the maximum of 4 in this case + */ + /* bits sorted this way (3...0): ECCS1,ECCS0,ECCSE1,ECCSE0 */ + return ((status & STATUS_ECC_MASK) >> 2) | + ((status2 & STATUS_ECC_MASK) >> 4); + + case GD5FXGQ4XA_STATUS_ECC_8_BITFLIPS: + return 8; + + case STATUS_ECC_UNCOR_ERROR: + return -EBADMSG; + + default: + break; + } + + return -EINVAL; +} + static const struct mtd_ooblayout_ops gd5fxgq4xa_ooblayout = { .ecc = gd5fxgq4xa_ooblayout_ecc, .free = gd5fxgq4xa_ooblayout_free, }; +static const struct mtd_ooblayout_ops gd5fxgq4uexxg_ooblayout = { + .ecc = gd5fxgq4uexxg_ooblayout_ecc, + .free = gd5fxgq4uexxg_ooblayout_free, +}; + static const struct spinand_info gigadevice_spinand_table[] = { SPINAND_INFO("GD5F1GQ4xA", 0xF1, NAND_MEMORG(1, 2048, 64, 64, 1024, 1, 1, 1), @@ -114,6 +188,15 @@ static const struct spinand_info gigadevice_spinand_table[] = { 0, SPINAND_ECCINFO(&gd5fxgq4xa_ooblayout, gd5fxgq4xa_ecc_get_status)), + SPINAND_INFO("GD5F1GQ4UExxG", 0xd1, + NAND_MEMORG(1, 2048, 128, 64, 1024, 1, 1, 1), + NAND_ECCREQ(8, 512), + SPINAND_INFO_OP_VARIANTS(&read_cache_variants, + &write_cache_variants, + &update_cache_variants), + 0, + SPINAND_ECCINFO(&gd5fxgq4uexxg_ooblayout, + gd5fxgq4uexxg_ecc_get_status)), }; static int gigadevice_spinand_detect(struct spinand_device *spinand) diff --git a/drivers/mtd/nand/spi/macronix.c b/drivers/mtd/nand/spi/macronix.c index 98f6b9c4b684..d16b57081c95 100644 --- a/drivers/mtd/nand/spi/macronix.c +++ b/drivers/mtd/nand/spi/macronix.c @@ -10,6 +10,7 @@ #include <linux/mtd/spinand.h> #define SPINAND_MFR_MACRONIX 0xC2 +#define MACRONIX_ECCSR_MASK 0x0F static SPINAND_OP_VARIANTS(read_cache_variants, SPINAND_PAGE_READ_FROM_CACHE_X4_OP(0, 1, NULL, 0), @@ -55,7 +56,12 @@ static int mx35lf1ge4ab_get_eccsr(struct spinand_device *spinand, u8 *eccsr) SPI_MEM_OP_DUMMY(1, 1), SPI_MEM_OP_DATA_IN(1, eccsr, 1)); - return spi_mem_exec_op(spinand->spimem, &op); + int ret = spi_mem_exec_op(spinand->spimem, &op); + if (ret) + return ret; + + *eccsr &= MACRONIX_ECCSR_MASK; + return 0; } static int mx35lf1ge4ab_ecc_get_status(struct spinand_device *spinand, diff --git a/drivers/mtd/nand/spi/toshiba.c b/drivers/mtd/nand/spi/toshiba.c index 081265557e70..db8021da45b5 100644 --- a/drivers/mtd/nand/spi/toshiba.c +++ b/drivers/mtd/nand/spi/toshiba.c @@ -25,19 +25,19 @@ static SPINAND_OP_VARIANTS(write_cache_variants, static SPINAND_OP_VARIANTS(update_cache_variants, SPINAND_PROG_LOAD(false, 0, NULL, 0)); -static int tc58cvg2s0h_ooblayout_ecc(struct mtd_info *mtd, int section, +static int tc58cxgxsx_ooblayout_ecc(struct mtd_info *mtd, int section, struct mtd_oob_region *region) { - if (section > 7) + if (section > 0) return -ERANGE; - region->offset = 128 + 16 * section; - region->length = 16; + region->offset = mtd->oobsize / 2; + region->length = mtd->oobsize / 2; return 0; } -static int tc58cvg2s0h_ooblayout_free(struct mtd_info *mtd, int section, +static int tc58cxgxsx_ooblayout_free(struct mtd_info *mtd, int section, struct mtd_oob_region *region) { if (section > 0) @@ -45,17 +45,17 @@ static int tc58cvg2s0h_ooblayout_free(struct mtd_info *mtd, int section, /* 2 bytes reserved for BBM */ region->offset = 2; - region->length = 126; + region->length = (mtd->oobsize / 2) - 2; return 0; } -static const struct mtd_ooblayout_ops tc58cvg2s0h_ooblayout = { - .ecc = tc58cvg2s0h_ooblayout_ecc, - .free = tc58cvg2s0h_ooblayout_free, +static const struct mtd_ooblayout_ops tc58cxgxsx_ooblayout = { + .ecc = tc58cxgxsx_ooblayout_ecc, + .free = tc58cxgxsx_ooblayout_free, }; -static int tc58cvg2s0h_ecc_get_status(struct spinand_device *spinand, +static int tc58cxgxsx_ecc_get_status(struct spinand_device *spinand, u8 status) { struct nand_device *nand = spinand_to_nand(spinand); @@ -94,15 +94,66 @@ static int tc58cvg2s0h_ecc_get_status(struct spinand_device *spinand, } static const struct spinand_info toshiba_spinand_table[] = { - SPINAND_INFO("TC58CVG2S0H", 0xCD, + /* 3.3V 1Gb */ + SPINAND_INFO("TC58CVG0S3", 0xC2, + NAND_MEMORG(1, 2048, 128, 64, 1024, 1, 1, 1), + NAND_ECCREQ(8, 512), + SPINAND_INFO_OP_VARIANTS(&read_cache_variants, + &write_cache_variants, + &update_cache_variants), + 0, + SPINAND_ECCINFO(&tc58cxgxsx_ooblayout, + tc58cxgxsx_ecc_get_status)), + /* 3.3V 2Gb */ + SPINAND_INFO("TC58CVG1S3", 0xCB, + NAND_MEMORG(1, 2048, 128, 64, 2048, 1, 1, 1), + NAND_ECCREQ(8, 512), + SPINAND_INFO_OP_VARIANTS(&read_cache_variants, + &write_cache_variants, + &update_cache_variants), + 0, + SPINAND_ECCINFO(&tc58cxgxsx_ooblayout, + tc58cxgxsx_ecc_get_status)), + /* 3.3V 4Gb */ + SPINAND_INFO("TC58CVG2S0", 0xCD, + NAND_MEMORG(1, 4096, 256, 64, 2048, 1, 1, 1), + NAND_ECCREQ(8, 512), + SPINAND_INFO_OP_VARIANTS(&read_cache_variants, + &write_cache_variants, + &update_cache_variants), + 0, + SPINAND_ECCINFO(&tc58cxgxsx_ooblayout, + tc58cxgxsx_ecc_get_status)), + /* 1.8V 1Gb */ + SPINAND_INFO("TC58CYG0S3", 0xB2, + NAND_MEMORG(1, 2048, 128, 64, 1024, 1, 1, 1), + NAND_ECCREQ(8, 512), + SPINAND_INFO_OP_VARIANTS(&read_cache_variants, + &write_cache_variants, + &update_cache_variants), + 0, + SPINAND_ECCINFO(&tc58cxgxsx_ooblayout, + tc58cxgxsx_ecc_get_status)), + /* 1.8V 2Gb */ + SPINAND_INFO("TC58CYG1S3", 0xBB, + NAND_MEMORG(1, 2048, 128, 64, 2048, 1, 1, 1), + NAND_ECCREQ(8, 512), + SPINAND_INFO_OP_VARIANTS(&read_cache_variants, + &write_cache_variants, + &update_cache_variants), + 0, + SPINAND_ECCINFO(&tc58cxgxsx_ooblayout, + tc58cxgxsx_ecc_get_status)), + /* 1.8V 4Gb */ + SPINAND_INFO("TC58CYG2S0", 0xBD, NAND_MEMORG(1, 4096, 256, 64, 2048, 1, 1, 1), NAND_ECCREQ(8, 512), SPINAND_INFO_OP_VARIANTS(&read_cache_variants, &write_cache_variants, &update_cache_variants), - SPINAND_HAS_QE_BIT, - SPINAND_ECCINFO(&tc58cvg2s0h_ooblayout, - tc58cvg2s0h_ecc_get_status)), + 0, + SPINAND_ECCINFO(&tc58cxgxsx_ooblayout, + tc58cxgxsx_ecc_get_status)), }; static int toshiba_spinand_detect(struct spinand_device *spinand) diff --git a/drivers/mtd/spi-nor/Kconfig b/drivers/mtd/spi-nor/Kconfig index 44fe8018733c..414e88749b1f 100644 --- a/drivers/mtd/spi-nor/Kconfig +++ b/drivers/mtd/spi-nor/Kconfig @@ -7,14 +7,6 @@ menuconfig MTD_SPI_NOR if MTD_SPI_NOR -config MTD_MT81xx_NOR - tristate "Mediatek MT81xx SPI NOR flash controller" - depends on HAS_IOMEM - help - This enables access to SPI NOR flash, using MT81xx SPI NOR flash - controller. This controller does not support generic SPI BUS, it only - supports SPI NOR Flash. - config MTD_SPI_NOR_USE_4K_SECTORS bool "Use small 4096 B erase sectors" default y @@ -66,6 +58,14 @@ config SPI_HISI_SFC help This enables support for hisilicon SPI-NOR flash controller. +config SPI_MTK_QUADSPI + tristate "MediaTek Quad SPI controller" + depends on HAS_IOMEM + help + This enables support for the Quad SPI controller in master mode. + This controller does not support generic SPI. It only supports + SPI NOR. + config SPI_NXP_SPIFI tristate "NXP SPI Flash Interface (SPIFI)" depends on OF && (ARCH_LPC18XX || COMPILE_TEST) diff --git a/drivers/mtd/spi-nor/Makefile b/drivers/mtd/spi-nor/Makefile index a552efd22958..4e4d4005d7a6 100644 --- a/drivers/mtd/spi-nor/Makefile +++ b/drivers/mtd/spi-nor/Makefile @@ -4,7 +4,7 @@ obj-$(CONFIG_SPI_ASPEED_SMC) += aspeed-smc.o obj-$(CONFIG_SPI_CADENCE_QUADSPI) += cadence-quadspi.o obj-$(CONFIG_SPI_FSL_QUADSPI) += fsl-quadspi.o obj-$(CONFIG_SPI_HISI_SFC) += hisi-sfc.o -obj-$(CONFIG_MTD_MT81xx_NOR) += mtk-quadspi.o +obj-$(CONFIG_SPI_MTK_QUADSPI) += mtk-quadspi.o obj-$(CONFIG_SPI_NXP_SPIFI) += nxp-spifi.o obj-$(CONFIG_SPI_INTEL_SPI) += intel-spi.o obj-$(CONFIG_SPI_INTEL_SPI_PCI) += intel-spi-pci.o diff --git a/drivers/mtd/spi-nor/cadence-quadspi.c b/drivers/mtd/spi-nor/cadence-quadspi.c index 04cedd3a2bf6..792628750eec 100644 --- a/drivers/mtd/spi-nor/cadence-quadspi.c +++ b/drivers/mtd/spi-nor/cadence-quadspi.c @@ -44,6 +44,12 @@ /* Quirks */ #define CQSPI_NEEDS_WR_DELAY BIT(0) +/* Capabilities mask */ +#define CQSPI_BASE_HWCAPS_MASK \ + (SNOR_HWCAPS_READ | SNOR_HWCAPS_READ_FAST | \ + SNOR_HWCAPS_READ_1_1_2 | SNOR_HWCAPS_READ_1_1_4 | \ + SNOR_HWCAPS_PP) + struct cqspi_st; struct cqspi_flash_pdata { @@ -93,6 +99,11 @@ struct cqspi_st { struct cqspi_flash_pdata f_pdata[CQSPI_MAX_CHIPSELECT]; }; +struct cqspi_driver_platdata { + u32 hwcaps_mask; + u8 quirks; +}; + /* Operation timeout value */ #define CQSPI_TIMEOUT_MS 500 #define CQSPI_READ_TIMEOUT_MS 10 @@ -101,6 +112,7 @@ struct cqspi_st { #define CQSPI_INST_TYPE_SINGLE 0 #define CQSPI_INST_TYPE_DUAL 1 #define CQSPI_INST_TYPE_QUAD 2 +#define CQSPI_INST_TYPE_OCTAL 3 #define CQSPI_DUMMY_CLKS_PER_BYTE 8 #define CQSPI_DUMMY_BYTES_MAX 4 @@ -418,9 +430,10 @@ static int cqspi_command_write(struct spi_nor *nor, const u8 opcode, void __iomem *reg_base = cqspi->iobase; unsigned int reg; unsigned int data; + u32 write_len; int ret; - if (n_tx > 4 || (n_tx && !txbuf)) { + if (n_tx > CQSPI_STIG_DATA_LEN_MAX || (n_tx && !txbuf)) { dev_err(nor->dev, "Invalid input argument, cmdlen %d txbuf 0x%p\n", n_tx, txbuf); @@ -433,10 +446,18 @@ static int cqspi_command_write(struct spi_nor *nor, const u8 opcode, reg |= ((n_tx - 1) & CQSPI_REG_CMDCTRL_WR_BYTES_MASK) << CQSPI_REG_CMDCTRL_WR_BYTES_LSB; data = 0; - memcpy(&data, txbuf, n_tx); + write_len = (n_tx > 4) ? 4 : n_tx; + memcpy(&data, txbuf, write_len); + txbuf += write_len; writel(data, reg_base + CQSPI_REG_CMDWRITEDATALOWER); - } + if (n_tx > 4) { + data = 0; + write_len = n_tx - 4; + memcpy(&data, txbuf, write_len); + writel(data, reg_base + CQSPI_REG_CMDWRITEDATAUPPER); + } + } ret = cqspi_exec_flash_cmd(cqspi, reg); return ret; } @@ -911,6 +932,9 @@ static int cqspi_set_protocol(struct spi_nor *nor, const int read) case SNOR_PROTO_1_1_4: f_pdata->data_width = CQSPI_INST_TYPE_QUAD; break; + case SNOR_PROTO_1_1_8: + f_pdata->data_width = CQSPI_INST_TYPE_OCTAL; + break; default: return -EINVAL; } @@ -1213,21 +1237,23 @@ static void cqspi_request_mmap_dma(struct cqspi_st *cqspi) static int cqspi_setup_flash(struct cqspi_st *cqspi, struct device_node *np) { - const struct spi_nor_hwcaps hwcaps = { - .mask = SNOR_HWCAPS_READ | - SNOR_HWCAPS_READ_FAST | - SNOR_HWCAPS_READ_1_1_2 | - SNOR_HWCAPS_READ_1_1_4 | - SNOR_HWCAPS_PP, - }; struct platform_device *pdev = cqspi->pdev; struct device *dev = &pdev->dev; + const struct cqspi_driver_platdata *ddata; + struct spi_nor_hwcaps hwcaps; struct cqspi_flash_pdata *f_pdata; struct spi_nor *nor; struct mtd_info *mtd; unsigned int cs; int i, ret; + ddata = of_device_get_match_data(dev); + if (!ddata) { + dev_err(dev, "Couldn't find driver data\n"); + return -EINVAL; + } + hwcaps.mask = ddata->hwcaps_mask; + /* Get flash device data */ for_each_available_child_of_node(dev->of_node, np) { ret = of_property_read_u32(np, "reg", &cs); @@ -1310,7 +1336,7 @@ static int cqspi_probe(struct platform_device *pdev) struct cqspi_st *cqspi; struct resource *res; struct resource *res_ahb; - unsigned long data; + const struct cqspi_driver_platdata *ddata; int ret; int irq; @@ -1377,8 +1403,8 @@ static int cqspi_probe(struct platform_device *pdev) } cqspi->master_ref_clk_hz = clk_get_rate(cqspi->clk); - data = (unsigned long)of_device_get_match_data(dev); - if (data & CQSPI_NEEDS_WR_DELAY) + ddata = of_device_get_match_data(dev); + if (ddata && (ddata->quirks & CQSPI_NEEDS_WR_DELAY)) cqspi->wr_delay = 5 * DIV_ROUND_UP(NSEC_PER_SEC, cqspi->master_ref_clk_hz); @@ -1460,14 +1486,32 @@ static const struct dev_pm_ops cqspi__dev_pm_ops = { #define CQSPI_DEV_PM_OPS NULL #endif +static const struct cqspi_driver_platdata cdns_qspi = { + .hwcaps_mask = CQSPI_BASE_HWCAPS_MASK, +}; + +static const struct cqspi_driver_platdata k2g_qspi = { + .hwcaps_mask = CQSPI_BASE_HWCAPS_MASK, + .quirks = CQSPI_NEEDS_WR_DELAY, +}; + +static const struct cqspi_driver_platdata am654_ospi = { + .hwcaps_mask = CQSPI_BASE_HWCAPS_MASK | SNOR_HWCAPS_READ_1_1_8, + .quirks = CQSPI_NEEDS_WR_DELAY, +}; + static const struct of_device_id cqspi_dt_ids[] = { { .compatible = "cdns,qspi-nor", - .data = (void *)0, + .data = &cdns_qspi, }, { .compatible = "ti,k2g-qspi", - .data = (void *)CQSPI_NEEDS_WR_DELAY, + .data = &k2g_qspi, + }, + { + .compatible = "ti,am654-ospi", + .data = &am654_ospi, }, { /* end of table */ } }; diff --git a/drivers/mtd/spi-nor/mtk-quadspi.c b/drivers/mtd/spi-nor/mtk-quadspi.c index 5442993b71ff..d9eed6844ba1 100644 --- a/drivers/mtd/spi-nor/mtk-quadspi.c +++ b/drivers/mtd/spi-nor/mtk-quadspi.c @@ -431,7 +431,8 @@ static int mtk_nor_init(struct mtk_nor *mtk_nor, struct device_node *flash_node) { const struct spi_nor_hwcaps hwcaps = { - .mask = SNOR_HWCAPS_READ_FAST | + .mask = SNOR_HWCAPS_READ | + SNOR_HWCAPS_READ_FAST | SNOR_HWCAPS_READ_1_1_2 | SNOR_HWCAPS_PP, }; diff --git a/drivers/mtd/spi-nor/spi-nor.c b/drivers/mtd/spi-nor/spi-nor.c index 6e13bbd1aaa5..fae147452aff 100644 --- a/drivers/mtd/spi-nor/spi-nor.c +++ b/drivers/mtd/spi-nor/spi-nor.c @@ -68,7 +68,7 @@ enum spi_nor_read_command_index { SNOR_CMD_READ_4_4_4, SNOR_CMD_READ_1_4_4_DTR, - /* Octo SPI */ + /* Octal SPI */ SNOR_CMD_READ_1_1_8, SNOR_CMD_READ_1_8_8, SNOR_CMD_READ_8_8_8, @@ -85,7 +85,7 @@ enum spi_nor_pp_command_index { SNOR_CMD_PP_1_4_4, SNOR_CMD_PP_4_4_4, - /* Octo SPI */ + /* Octal SPI */ SNOR_CMD_PP_1_1_8, SNOR_CMD_PP_1_8_8, SNOR_CMD_PP_8_8_8, @@ -278,6 +278,7 @@ struct flash_info { #define NO_CHIP_ERASE BIT(12) /* Chip does not support chip erase */ #define SPI_NOR_SKIP_SFDP BIT(13) /* Skip parsing of SFDP tables */ #define USE_CLSR BIT(14) /* use CLSR command */ +#define SPI_NOR_OCTAL_READ BIT(15) /* Flash supports Octal Read */ /* Part specific fixup hooks. */ const struct spi_nor_fixups *fixups; @@ -398,6 +399,8 @@ static u8 spi_nor_convert_3to4_read(u8 opcode) { SPINOR_OP_READ_1_2_2, SPINOR_OP_READ_1_2_2_4B }, { SPINOR_OP_READ_1_1_4, SPINOR_OP_READ_1_1_4_4B }, { SPINOR_OP_READ_1_4_4, SPINOR_OP_READ_1_4_4_4B }, + { SPINOR_OP_READ_1_1_8, SPINOR_OP_READ_1_1_8_4B }, + { SPINOR_OP_READ_1_8_8, SPINOR_OP_READ_1_8_8_4B }, { SPINOR_OP_READ_1_1_1_DTR, SPINOR_OP_READ_1_1_1_DTR_4B }, { SPINOR_OP_READ_1_2_2_DTR, SPINOR_OP_READ_1_2_2_DTR_4B }, @@ -414,6 +417,8 @@ static u8 spi_nor_convert_3to4_program(u8 opcode) { SPINOR_OP_PP, SPINOR_OP_PP_4B }, { SPINOR_OP_PP_1_1_4, SPINOR_OP_PP_1_1_4_4B }, { SPINOR_OP_PP_1_4_4, SPINOR_OP_PP_1_4_4_4B }, + { SPINOR_OP_PP_1_1_8, SPINOR_OP_PP_1_1_8_4B }, + { SPINOR_OP_PP_1_8_8, SPINOR_OP_PP_1_8_8_4B }, }; return spi_nor_convert_opcode(opcode, spi_nor_3to4_program, @@ -1740,7 +1745,11 @@ static const struct flash_info spi_nor_ids[] = { { "en25q32b", INFO(0x1c3016, 0, 64 * 1024, 64, 0) }, { "en25p64", INFO(0x1c2017, 0, 64 * 1024, 128, 0) }, { "en25q64", INFO(0x1c3017, 0, 64 * 1024, 128, SECT_4K) }, + { "en25q80a", INFO(0x1c3014, 0, 64 * 1024, 16, + SECT_4K | SPI_NOR_DUAL_READ) }, { "en25qh32", INFO(0x1c7016, 0, 64 * 1024, 64, 0) }, + { "en25qh64", INFO(0x1c7017, 0, 64 * 1024, 128, + SECT_4K | SPI_NOR_DUAL_READ) }, { "en25qh128", INFO(0x1c7018, 0, 64 * 1024, 256, 0) }, { "en25qh256", INFO(0x1c7019, 0, 64 * 1024, 512, 0) }, { "en25s64", INFO(0x1c3817, 0, 64 * 1024, 128, SECT_4K) }, @@ -1836,6 +1845,8 @@ static const struct flash_info spi_nor_ids[] = { { "mx25l3255e", INFO(0xc29e16, 0, 64 * 1024, 64, SECT_4K) }, { "mx25l6405d", INFO(0xc22017, 0, 64 * 1024, 128, SECT_4K) }, { "mx25u2033e", INFO(0xc22532, 0, 64 * 1024, 4, SECT_4K) }, + { "mx25u3235f", INFO(0xc22536, 0, 64 * 1024, 64, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, { "mx25u4035", INFO(0xc22533, 0, 64 * 1024, 8, SECT_4K) }, { "mx25u8035", INFO(0xc22534, 0, 64 * 1024, 16, SECT_4K) }, { "mx25u6435f", INFO(0xc22537, 0, 64 * 1024, 128, SECT_4K) }, @@ -1847,6 +1858,8 @@ static const struct flash_info spi_nor_ids[] = { SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) .fixups = &mx25l25635_fixups }, { "mx25u25635f", INFO(0xc22539, 0, 64 * 1024, 512, SECT_4K | SPI_NOR_4B_OPCODES) }, + { "mx25v8035f", INFO(0xc22314, 0, 64 * 1024, 16, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, { "mx25l25655e", INFO(0xc22619, 0, 64 * 1024, 512, 0) }, { "mx66l51235l", INFO(0xc2201a, 0, 64 * 1024, 1024, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | SPI_NOR_4B_OPCODES) }, { "mx66u51235f", INFO(0xc2253a, 0, 64 * 1024, 1024, SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | SPI_NOR_4B_OPCODES) }, @@ -1872,7 +1885,8 @@ static const struct flash_info spi_nor_ids[] = { /* Micron */ { "mt35xu512aba", INFO(0x2c5b1a, 0, 128 * 1024, 512, - SECT_4K | USE_FSR | SPI_NOR_4B_OPCODES) + SECT_4K | USE_FSR | SPI_NOR_OCTAL_READ | + SPI_NOR_4B_OPCODES) }, /* PMC */ @@ -1885,13 +1899,17 @@ static const struct flash_info spi_nor_ids[] = { */ { "s25sl032p", INFO(0x010215, 0x4d00, 64 * 1024, 64, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, { "s25sl064p", INFO(0x010216, 0x4d00, 64 * 1024, 128, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, + { "s25fl128s0", INFO6(0x012018, 0x4d0080, 256 * 1024, 64, + SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | USE_CLSR) }, + { "s25fl128s1", INFO6(0x012018, 0x4d0180, 64 * 1024, 256, + SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | USE_CLSR) }, { "s25fl256s0", INFO(0x010219, 0x4d00, 256 * 1024, 128, USE_CLSR) }, { "s25fl256s1", INFO(0x010219, 0x4d01, 64 * 1024, 512, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | USE_CLSR) }, - { "s25fl512s", INFO(0x010220, 0x4d00, 256 * 1024, 256, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | USE_CLSR) }, + { "s25fl512s", INFO6(0x010220, 0x4d0080, 256 * 1024, 256, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | USE_CLSR) }, + { "s25fs512s", INFO6(0x010220, 0x4d0081, 256 * 1024, 256, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | USE_CLSR) }, { "s70fl01gs", INFO(0x010221, 0x4d00, 256 * 1024, 256, 0) }, { "s25sl12800", INFO(0x012018, 0x0300, 256 * 1024, 64, 0) }, { "s25sl12801", INFO(0x012018, 0x0301, 64 * 1024, 256, 0) }, - { "s25fl128s", INFO6(0x012018, 0x4d0180, 64 * 1024, 256, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | USE_CLSR) }, { "s25fl129p0", INFO(0x012018, 0x4d00, 256 * 1024, 64, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | USE_CLSR) }, { "s25fl129p1", INFO(0x012018, 0x4d01, 64 * 1024, 256, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | USE_CLSR) }, { "s25sl004a", INFO(0x010212, 0, 64 * 1024, 8, 0) }, @@ -3591,6 +3609,13 @@ static int spi_nor_init_params(struct spi_nor *nor, SNOR_PROTO_1_1_4); } + if (info->flags & SPI_NOR_OCTAL_READ) { + params->hwcaps.mask |= SNOR_HWCAPS_READ_1_1_8; + spi_nor_set_read_settings(¶ms->reads[SNOR_CMD_READ_1_1_8], + 0, 8, SPINOR_OP_READ_1_1_8, + SNOR_PROTO_1_1_8); + } + /* Page Program settings. */ params->hwcaps.mask |= SNOR_HWCAPS_PP; spi_nor_set_pp_settings(¶ms->page_programs[SNOR_CMD_PP], diff --git a/drivers/net/dsa/lantiq_gswip.c b/drivers/net/dsa/lantiq_gswip.c index 693a67f45bef..ddc1f9ca8ebc 100644 --- a/drivers/net/dsa/lantiq_gswip.c +++ b/drivers/net/dsa/lantiq_gswip.c @@ -1162,6 +1162,12 @@ static struct platform_driver gswip_driver = { module_platform_driver(gswip_driver); +MODULE_FIRMWARE("lantiq/xrx300_phy11g_a21.bin"); +MODULE_FIRMWARE("lantiq/xrx300_phy22f_a21.bin"); +MODULE_FIRMWARE("lantiq/xrx200_phy11g_a14.bin"); +MODULE_FIRMWARE("lantiq/xrx200_phy11g_a22.bin"); +MODULE_FIRMWARE("lantiq/xrx200_phy22f_a14.bin"); +MODULE_FIRMWARE("lantiq/xrx200_phy22f_a22.bin"); MODULE_AUTHOR("Hauke Mehrtens <hauke@hauke-m.de>"); MODULE_DESCRIPTION("Lantiq / Intel GSWIP driver"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/net/dsa/mv88e6xxx/chip.c b/drivers/net/dsa/mv88e6xxx/chip.c index 12fd7ce3f1ff..7e3c00bd9532 100644 --- a/drivers/net/dsa/mv88e6xxx/chip.c +++ b/drivers/net/dsa/mv88e6xxx/chip.c @@ -896,7 +896,7 @@ static uint64_t _mv88e6xxx_get_ethtool_stat(struct mv88e6xxx_chip *chip, default: return U64_MAX; } - value = (((u64)high) << 16) | low; + value = (((u64)high) << 32) | low; return value; } @@ -3093,7 +3093,7 @@ static const struct mv88e6xxx_ops mv88e6161_ops = { .port_disable_pri_override = mv88e6xxx_port_disable_pri_override, .port_link_state = mv88e6352_port_link_state, .port_get_cmode = mv88e6185_port_get_cmode, - .stats_snapshot = mv88e6320_g1_stats_snapshot, + .stats_snapshot = mv88e6xxx_g1_stats_snapshot, .stats_set_histogram = mv88e6095_g1_stats_set_histogram, .stats_get_sset_count = mv88e6095_stats_get_sset_count, .stats_get_strings = mv88e6095_stats_get_strings, @@ -4595,6 +4595,14 @@ static int mv88e6xxx_smi_init(struct mv88e6xxx_chip *chip, return 0; } +static void mv88e6xxx_ports_cmode_init(struct mv88e6xxx_chip *chip) +{ + int i; + + for (i = 0; i < mv88e6xxx_num_ports(chip); i++) + chip->ports[i].cmode = MV88E6XXX_PORT_STS_CMODE_INVALID; +} + static enum dsa_tag_protocol mv88e6xxx_get_tag_protocol(struct dsa_switch *ds, int port) { @@ -4631,6 +4639,8 @@ static const char *mv88e6xxx_drv_probe(struct device *dsa_dev, if (err) goto free; + mv88e6xxx_ports_cmode_init(chip); + mutex_lock(&chip->reg_lock); err = mv88e6xxx_switch_reset(chip); mutex_unlock(&chip->reg_lock); diff --git a/drivers/net/dsa/mv88e6xxx/port.c b/drivers/net/dsa/mv88e6xxx/port.c index ebd26b6a93e6..79ab51e69aee 100644 --- a/drivers/net/dsa/mv88e6xxx/port.c +++ b/drivers/net/dsa/mv88e6xxx/port.c @@ -398,6 +398,10 @@ int mv88e6390x_port_set_cmode(struct mv88e6xxx_chip *chip, int port, cmode = 0; } + /* cmode doesn't change, nothing to do for us */ + if (cmode == chip->ports[port].cmode) + return 0; + lane = mv88e6390x_serdes_get_lane(chip, port); if (lane < 0) return lane; @@ -408,7 +412,7 @@ int mv88e6390x_port_set_cmode(struct mv88e6xxx_chip *chip, int port, return err; } - err = mv88e6390_serdes_power(chip, port, false); + err = mv88e6390x_serdes_power(chip, port, false); if (err) return err; @@ -424,7 +428,7 @@ int mv88e6390x_port_set_cmode(struct mv88e6xxx_chip *chip, int port, if (err) return err; - err = mv88e6390_serdes_power(chip, port, true); + err = mv88e6390x_serdes_power(chip, port, true); if (err) return err; diff --git a/drivers/net/dsa/mv88e6xxx/port.h b/drivers/net/dsa/mv88e6xxx/port.h index e583641de758..4aadf321edb7 100644 --- a/drivers/net/dsa/mv88e6xxx/port.h +++ b/drivers/net/dsa/mv88e6xxx/port.h @@ -52,6 +52,7 @@ #define MV88E6185_PORT_STS_CMODE_1000BASE_X 0x0005 #define MV88E6185_PORT_STS_CMODE_PHY 0x0006 #define MV88E6185_PORT_STS_CMODE_DISABLED 0x0007 +#define MV88E6XXX_PORT_STS_CMODE_INVALID 0xff /* Offset 0x01: MAC (or PCS or Physical) Control Register */ #define MV88E6XXX_PORT_MAC_CTL 0x01 diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0.c b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0.c index b58ca7cb8e9d..fbba300c1d01 100644 --- a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0.c +++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0.c @@ -275,6 +275,9 @@ static int hw_atl_b0_hw_offload_set(struct aq_hw_s *self, static int hw_atl_b0_hw_init_tx_path(struct aq_hw_s *self) { + /* Tx TC/Queue number config */ + hw_atl_rpb_tps_tx_tc_mode_set(self, 1U); + hw_atl_thm_lso_tcp_flag_of_first_pkt_set(self, 0x0FF6U); hw_atl_thm_lso_tcp_flag_of_middle_pkt_set(self, 0x0FF6U); hw_atl_thm_lso_tcp_flag_of_last_pkt_set(self, 0x0F7FU); diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_llh.c b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_llh.c index 939f77e2e117..8ac7a67b15c1 100644 --- a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_llh.c +++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_llh.c @@ -1274,6 +1274,15 @@ void hw_atl_tpb_tx_buff_en_set(struct aq_hw_s *aq_hw, u32 tx_buff_en) HW_ATL_TPB_TX_BUF_EN_SHIFT, tx_buff_en); } +void hw_atl_rpb_tps_tx_tc_mode_set(struct aq_hw_s *aq_hw, + u32 tx_traf_class_mode) +{ + aq_hw_write_reg_bit(aq_hw, HW_ATL_TPB_TX_TC_MODE_ADDR, + HW_ATL_TPB_TX_TC_MODE_MSK, + HW_ATL_TPB_TX_TC_MODE_SHIFT, + tx_traf_class_mode); +} + void hw_atl_tpb_tx_buff_hi_threshold_per_tc_set(struct aq_hw_s *aq_hw, u32 tx_buff_hi_threshold_per_tc, u32 buffer) diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_llh.h b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_llh.h index 03c570d115fe..f529540bfd7e 100644 --- a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_llh.h +++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_llh.h @@ -605,6 +605,10 @@ void hw_atl_thm_lso_tcp_flag_of_middle_pkt_set(struct aq_hw_s *aq_hw, /* tpb */ +/* set TX Traffic Class Mode */ +void hw_atl_rpb_tps_tx_tc_mode_set(struct aq_hw_s *aq_hw, + u32 tx_traf_class_mode); + /* set tx buffer enable */ void hw_atl_tpb_tx_buff_en_set(struct aq_hw_s *aq_hw, u32 tx_buff_en); diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_llh_internal.h b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_llh_internal.h index 8470d92db812..e91ffce005f1 100644 --- a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_llh_internal.h +++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_llh_internal.h @@ -1948,6 +1948,19 @@ /* default value of bitfield tx_buf_en */ #define HW_ATL_TPB_TX_BUF_EN_DEFAULT 0x0 +/* register address for bitfield tx_tc_mode */ +#define HW_ATL_TPB_TX_TC_MODE_ADDR 0x00007900 +/* bitmask for bitfield tx_tc_mode */ +#define HW_ATL_TPB_TX_TC_MODE_MSK 0x00000100 +/* inverted bitmask for bitfield tx_tc_mode */ +#define HW_ATL_TPB_TX_TC_MODE_MSKN 0xFFFFFEFF +/* lower bit position of bitfield tx_tc_mode */ +#define HW_ATL_TPB_TX_TC_MODE_SHIFT 8 +/* width of bitfield tx_tc_mode */ +#define HW_ATL_TPB_TX_TC_MODE_WIDTH 1 +/* default value of bitfield tx_tc_mode */ +#define HW_ATL_TPB_TX_TC_MODE_DEFAULT 0x0 + /* tx tx{b}_hi_thresh[c:0] bitfield definitions * preprocessor definitions for the bitfield "tx{b}_hi_thresh[c:0]". * parameter: buffer {b} | stride size 0x10 | range [0, 7] diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index d95730c6e0f2..803f7990d32b 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -500,6 +500,12 @@ normal_tx: } length >>= 9; + if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) { + dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n", + skb->len); + i = 0; + goto tx_dma_error; + } flags |= bnxt_lhint_arr[length]; txbd->tx_bd_len_flags_type = cpu_to_le32(flags); diff --git a/drivers/net/ethernet/microchip/enc28j60.c b/drivers/net/ethernet/microchip/enc28j60.c index f6ecfa778660..8f72587b5a2c 100644 --- a/drivers/net/ethernet/microchip/enc28j60.c +++ b/drivers/net/ethernet/microchip/enc28j60.c @@ -1681,5 +1681,5 @@ MODULE_DESCRIPTION(DRV_NAME " ethernet driver"); MODULE_AUTHOR("Claudio Lanconelli <lanconelli.claudio@eptar.com>"); MODULE_LICENSE("GPL"); module_param_named(debug, debug.msg_enable, int, 0); -MODULE_PARM_DESC(debug, "Debug verbosity level (0=none, ..., ffff=all)"); +MODULE_PARM_DESC(debug, "Debug verbosity level in amount of bits set (0=none, ..., 31=all)"); MODULE_ALIAS("spi:" DRV_NAME); diff --git a/drivers/net/ethernet/microchip/lan743x_main.c b/drivers/net/ethernet/microchip/lan743x_main.c index 310807ef328b..4d1b4a24907f 100644 --- a/drivers/net/ethernet/microchip/lan743x_main.c +++ b/drivers/net/ethernet/microchip/lan743x_main.c @@ -1400,7 +1400,8 @@ static int lan743x_tx_frame_start(struct lan743x_tx *tx, } static void lan743x_tx_frame_add_lso(struct lan743x_tx *tx, - unsigned int frame_length) + unsigned int frame_length, + int nr_frags) { /* called only from within lan743x_tx_xmit_frame. * assuming tx->ring_lock has already been acquired. @@ -1410,6 +1411,10 @@ static void lan743x_tx_frame_add_lso(struct lan743x_tx *tx, /* wrap up previous descriptor */ tx->frame_data0 |= TX_DESC_DATA0_EXT_; + if (nr_frags <= 0) { + tx->frame_data0 |= TX_DESC_DATA0_LS_; + tx->frame_data0 |= TX_DESC_DATA0_IOC_; + } tx_descriptor = &tx->ring_cpu_ptr[tx->frame_tail]; tx_descriptor->data0 = tx->frame_data0; @@ -1514,8 +1519,11 @@ static void lan743x_tx_frame_end(struct lan743x_tx *tx, u32 tx_tail_flags = 0; /* wrap up previous descriptor */ - tx->frame_data0 |= TX_DESC_DATA0_LS_; - tx->frame_data0 |= TX_DESC_DATA0_IOC_; + if ((tx->frame_data0 & TX_DESC_DATA0_DTYPE_MASK_) == + TX_DESC_DATA0_DTYPE_DATA_) { + tx->frame_data0 |= TX_DESC_DATA0_LS_; + tx->frame_data0 |= TX_DESC_DATA0_IOC_; + } tx_descriptor = &tx->ring_cpu_ptr[tx->frame_tail]; buffer_info = &tx->buffer_info[tx->frame_tail]; @@ -1600,7 +1608,7 @@ static netdev_tx_t lan743x_tx_xmit_frame(struct lan743x_tx *tx, } if (gso) - lan743x_tx_frame_add_lso(tx, frame_length); + lan743x_tx_frame_add_lso(tx, frame_length, nr_frags); if (nr_frags <= 0) goto finish; diff --git a/drivers/net/geneve.c b/drivers/net/geneve.c index 3377ac66a347..5583d993480d 100644 --- a/drivers/net/geneve.c +++ b/drivers/net/geneve.c @@ -692,15 +692,20 @@ out: static int geneve_open(struct net_device *dev) { struct geneve_dev *geneve = netdev_priv(dev); - bool ipv6 = !!(geneve->info.mode & IP_TUNNEL_INFO_IPV6); bool metadata = geneve->collect_md; + bool ipv4, ipv6; int ret = 0; + ipv6 = geneve->info.mode & IP_TUNNEL_INFO_IPV6 || metadata; + ipv4 = !ipv6 || metadata; #if IS_ENABLED(CONFIG_IPV6) - if (ipv6 || metadata) + if (ipv6) { ret = geneve_sock_add(geneve, true); + if (ret < 0 && ret != -EAFNOSUPPORT) + ipv4 = false; + } #endif - if (!ret && (!ipv6 || metadata)) + if (ipv4) ret = geneve_sock_add(geneve, false); if (ret < 0) geneve_sock_release(geneve); diff --git a/drivers/net/hyperv/netvsc_drv.c b/drivers/net/hyperv/netvsc_drv.c index 256adbd044f5..cf4897043e83 100644 --- a/drivers/net/hyperv/netvsc_drv.c +++ b/drivers/net/hyperv/netvsc_drv.c @@ -744,6 +744,14 @@ void netvsc_linkstatus_callback(struct net_device *net, schedule_delayed_work(&ndev_ctx->dwork, 0); } +static void netvsc_comp_ipcsum(struct sk_buff *skb) +{ + struct iphdr *iph = (struct iphdr *)skb->data; + + iph->check = 0; + iph->check = ip_fast_csum(iph, iph->ihl); +} + static struct sk_buff *netvsc_alloc_recv_skb(struct net_device *net, struct netvsc_channel *nvchan) { @@ -770,9 +778,17 @@ static struct sk_buff *netvsc_alloc_recv_skb(struct net_device *net, /* skb is already created with CHECKSUM_NONE */ skb_checksum_none_assert(skb); - /* - * In Linux, the IP checksum is always checked. - * Do L4 checksum offload if enabled and present. + /* Incoming packets may have IP header checksum verified by the host. + * They may not have IP header checksum computed after coalescing. + * We compute it here if the flags are set, because on Linux, the IP + * checksum is always checked. + */ + if (csum_info && csum_info->receive.ip_checksum_value_invalid && + csum_info->receive.ip_checksum_succeeded && + skb->protocol == htons(ETH_P_IP)) + netvsc_comp_ipcsum(skb); + + /* Do L4 checksum offload if enabled and present. */ if (csum_info && (net->features & NETIF_F_RXCSUM)) { if (csum_info->receive.tcp_checksum_succeeded || diff --git a/drivers/net/phy/dp83867.c b/drivers/net/phy/dp83867.c index da6a67d47ce9..56fa3606cb9c 100644 --- a/drivers/net/phy/dp83867.c +++ b/drivers/net/phy/dp83867.c @@ -19,6 +19,7 @@ #include <linux/module.h> #include <linux/of.h> #include <linux/phy.h> +#include <linux/delay.h> #include <dt-bindings/net/ti-dp83867.h> @@ -325,6 +326,8 @@ static int dp83867_phy_reset(struct phy_device *phydev) if (err < 0) return err; + usleep_range(10, 20); + return dp83867_config_init(phydev); } diff --git a/drivers/net/phy/micrel.c b/drivers/net/phy/micrel.c index b1f959935f50..b7df0295a3ca 100644 --- a/drivers/net/phy/micrel.c +++ b/drivers/net/phy/micrel.c @@ -344,6 +344,17 @@ static int ksz8041_config_aneg(struct phy_device *phydev) return genphy_config_aneg(phydev); } +static int ksz8061_config_init(struct phy_device *phydev) +{ + int ret; + + ret = phy_write_mmd(phydev, MDIO_MMD_PMAPMD, MDIO_DEVID1, 0xB61A); + if (ret) + return ret; + + return kszphy_config_init(phydev); +} + static int ksz9021_load_values_from_of(struct phy_device *phydev, const struct device_node *of_node, u16 reg, @@ -1040,7 +1051,7 @@ static struct phy_driver ksphy_driver[] = { .name = "Micrel KSZ8061", .phy_id_mask = MICREL_PHY_ID_MASK, .features = PHY_BASIC_FEATURES, - .config_init = kszphy_config_init, + .config_init = ksz8061_config_init, .ack_interrupt = kszphy_ack_interrupt, .config_intr = kszphy_config_intr, .suspend = genphy_suspend, diff --git a/drivers/net/phy/phylink.c b/drivers/net/phy/phylink.c index 938803237d7f..85987aac31c4 100644 --- a/drivers/net/phy/phylink.c +++ b/drivers/net/phy/phylink.c @@ -320,6 +320,10 @@ static int phylink_get_mac_state(struct phylink *pl, struct phylink_link_state * linkmode_zero(state->lp_advertising); state->interface = pl->link_config.interface; state->an_enabled = pl->link_config.an_enabled; + state->speed = SPEED_UNKNOWN; + state->duplex = DUPLEX_UNKNOWN; + state->pause = MLO_PAUSE_NONE; + state->an_complete = 0; state->link = 1; return pl->ops->mac_link_state(ndev, state); diff --git a/drivers/net/tun.c b/drivers/net/tun.c index fed298c0cb39..53f4f37b0ffd 100644 --- a/drivers/net/tun.c +++ b/drivers/net/tun.c @@ -2167,9 +2167,9 @@ static void *tun_ring_recv(struct tun_file *tfile, int noblock, int *err) } add_wait_queue(&tfile->wq.wait, &wait); - current->state = TASK_INTERRUPTIBLE; while (1) { + set_current_state(TASK_INTERRUPTIBLE); ptr = ptr_ring_consume(&tfile->tx_ring); if (ptr) break; @@ -2185,7 +2185,7 @@ static void *tun_ring_recv(struct tun_file *tfile, int noblock, int *err) schedule(); } - current->state = TASK_RUNNING; + __set_current_state(TASK_RUNNING); remove_wait_queue(&tfile->wq.wait, &wait); out: diff --git a/drivers/net/xen-netback/hash.c b/drivers/net/xen-netback/hash.c index 0ccb021f1e78..10d580c3dea3 100644 --- a/drivers/net/xen-netback/hash.c +++ b/drivers/net/xen-netback/hash.c @@ -454,6 +454,8 @@ void xenvif_init_hash(struct xenvif *vif) if (xenvif_hash_cache_size == 0) return; + BUG_ON(vif->hash.cache.count); + spin_lock_init(&vif->hash.cache.lock); INIT_LIST_HEAD(&vif->hash.cache.list); } diff --git a/drivers/net/xen-netback/interface.c b/drivers/net/xen-netback/interface.c index 182d6770f102..6da12518e693 100644 --- a/drivers/net/xen-netback/interface.c +++ b/drivers/net/xen-netback/interface.c @@ -153,6 +153,13 @@ static u16 xenvif_select_queue(struct net_device *dev, struct sk_buff *skb, { struct xenvif *vif = netdev_priv(dev); unsigned int size = vif->hash.size; + unsigned int num_queues; + + /* If queues are not set up internally - always return 0 + * as the packet going to be dropped anyway */ + num_queues = READ_ONCE(vif->num_queues); + if (num_queues < 1) + return 0; if (vif->hash.alg == XEN_NETIF_CTRL_HASH_ALGORITHM_NONE) return fallback(dev, skb, NULL) % dev->real_num_tx_queues; diff --git a/drivers/net/xen-netback/netback.c b/drivers/net/xen-netback/netback.c index 80aae3a32c2a..f09948b009dd 100644 --- a/drivers/net/xen-netback/netback.c +++ b/drivers/net/xen-netback/netback.c @@ -1072,11 +1072,6 @@ static int xenvif_handle_frag_list(struct xenvif_queue *queue, struct sk_buff *s skb_frag_size_set(&frags[i], len); } - /* Copied all the bits from the frag list -- free it. */ - skb_frag_list_init(skb); - xenvif_skb_zerocopy_prepare(queue, nskb); - kfree_skb(nskb); - /* Release all the original (foreign) frags. */ for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) skb_frag_unref(skb, f); @@ -1145,6 +1140,8 @@ static int xenvif_tx_submit(struct xenvif_queue *queue) xenvif_fill_frags(queue, skb); if (unlikely(skb_has_frag_list(skb))) { + struct sk_buff *nskb = skb_shinfo(skb)->frag_list; + xenvif_skb_zerocopy_prepare(queue, nskb); if (xenvif_handle_frag_list(queue, skb)) { if (net_ratelimit()) netdev_err(queue->vif->dev, @@ -1153,6 +1150,9 @@ static int xenvif_tx_submit(struct xenvif_queue *queue) kfree_skb(skb); continue; } + /* Copied all the bits from the frag list -- free it. */ + skb_frag_list_init(skb); + kfree_skb(nskb); } skb->dev = queue->vif->dev; diff --git a/drivers/scsi/3w-9xxx.c b/drivers/scsi/3w-9xxx.c index a3c20e3a8b7c..3337b1e80412 100644 --- a/drivers/scsi/3w-9xxx.c +++ b/drivers/scsi/3w-9xxx.c @@ -2009,7 +2009,7 @@ static int twa_probe(struct pci_dev *pdev, const struct pci_device_id *dev_id) struct Scsi_Host *host = NULL; TW_Device_Extension *tw_dev; unsigned long mem_addr, mem_len; - int retval = -ENODEV; + int retval; retval = pci_enable_device(pdev); if (retval) { @@ -2020,8 +2020,10 @@ static int twa_probe(struct pci_dev *pdev, const struct pci_device_id *dev_id) pci_set_master(pdev); pci_try_set_mwi(pdev); - if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) || - dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32))) { + retval = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (retval) + retval = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (retval) { TW_PRINTK(host, TW_DRIVER, 0x23, "Failed to set dma mask"); retval = -ENODEV; goto out_disable_device; @@ -2240,8 +2242,10 @@ static int twa_resume(struct pci_dev *pdev) pci_set_master(pdev); pci_try_set_mwi(pdev); - if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) || - dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32))) { + retval = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (retval) + retval = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (retval) { TW_PRINTK(host, TW_DRIVER, 0x40, "Failed to set dma mask during resume"); retval = -ENODEV; goto out_disable_device; diff --git a/drivers/scsi/3w-sas.c b/drivers/scsi/3w-sas.c index cd096104bcec..dda6fa857709 100644 --- a/drivers/scsi/3w-sas.c +++ b/drivers/scsi/3w-sas.c @@ -1573,8 +1573,10 @@ static int twl_probe(struct pci_dev *pdev, const struct pci_device_id *dev_id) pci_set_master(pdev); pci_try_set_mwi(pdev); - if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) || - dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32))) { + retval = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (retval) + retval = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (retval) { TW_PRINTK(host, TW_DRIVER, 0x18, "Failed to set dma mask"); retval = -ENODEV; goto out_disable_device; @@ -1805,8 +1807,10 @@ static int twl_resume(struct pci_dev *pdev) pci_set_master(pdev); pci_try_set_mwi(pdev); - if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) || - dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32))) { + retval = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (retval) + retval = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (retval) { TW_PRINTK(host, TW_DRIVER, 0x25, "Failed to set dma mask during resume"); retval = -ENODEV; goto out_disable_device; diff --git a/drivers/scsi/aic94xx/aic94xx_init.c b/drivers/scsi/aic94xx/aic94xx_init.c index 07efcb9b5b94..bbdae67774f0 100644 --- a/drivers/scsi/aic94xx/aic94xx_init.c +++ b/drivers/scsi/aic94xx/aic94xx_init.c @@ -769,9 +769,11 @@ static int asd_pci_probe(struct pci_dev *dev, const struct pci_device_id *id) if (err) goto Err_remove; - err = -ENODEV; - if (dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(64)) || - dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(32))) { + err = dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(64)); + if (err) + err = dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(32)); + if (err) { + err = -ENODEV; asd_printk("no suitable DMA mask for %s\n", pci_name(dev)); goto Err_remove; } diff --git a/drivers/scsi/bfa/bfad.c b/drivers/scsi/bfa/bfad.c index 42a0caf6740d..88880a66a189 100644 --- a/drivers/scsi/bfa/bfad.c +++ b/drivers/scsi/bfa/bfad.c @@ -727,7 +727,7 @@ bfad_init_timer(struct bfad_s *bfad) int bfad_pci_init(struct pci_dev *pdev, struct bfad_s *bfad) { - int rc = -ENODEV; + int rc = -ENODEV; if (pci_enable_device(pdev)) { printk(KERN_ERR "pci_enable_device fail %p\n", pdev); @@ -739,8 +739,12 @@ bfad_pci_init(struct pci_dev *pdev, struct bfad_s *bfad) pci_set_master(pdev); - if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) || - dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32))) { + rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (rc) + rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + + if (rc) { + rc = -ENODEV; printk(KERN_ERR "dma_set_mask_and_coherent fail %p\n", pdev); goto out_release_region; } @@ -1534,6 +1538,7 @@ bfad_pci_slot_reset(struct pci_dev *pdev) { struct bfad_s *bfad = pci_get_drvdata(pdev); u8 byte; + int rc; dev_printk(KERN_ERR, &pdev->dev, "bfad_pci_slot_reset flags: 0x%x\n", bfad->bfad_flags); @@ -1561,8 +1566,11 @@ bfad_pci_slot_reset(struct pci_dev *pdev) pci_save_state(pdev); pci_set_master(pdev); - if (dma_set_mask_and_coherent(&bfad->pcidev->dev, DMA_BIT_MASK(64)) || - dma_set_mask_and_coherent(&bfad->pcidev->dev, DMA_BIT_MASK(32))) + rc = dma_set_mask_and_coherent(&bfad->pcidev->dev, DMA_BIT_MASK(64)); + if (rc) + rc = dma_set_mask_and_coherent(&bfad->pcidev->dev, + DMA_BIT_MASK(32)); + if (rc) goto out_disable_device; if (restart_bfa(bfad) == -1) diff --git a/drivers/scsi/csiostor/csio_init.c b/drivers/scsi/csiostor/csio_init.c index cf629380a981..616b25bf7941 100644 --- a/drivers/scsi/csiostor/csio_init.c +++ b/drivers/scsi/csiostor/csio_init.c @@ -210,8 +210,11 @@ csio_pci_init(struct pci_dev *pdev, int *bars) pci_set_master(pdev); pci_try_set_mwi(pdev); - if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) || - dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32))) { + rv = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (rv) + rv = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (rv) { + rv = -ENODEV; dev_err(&pdev->dev, "No suitable DMA available.\n"); goto err_release_regions; } diff --git a/drivers/scsi/hisi_sas/hisi_sas_main.c b/drivers/scsi/hisi_sas/hisi_sas_main.c index eed7fc5b3389..bc17fa0d8375 100644 --- a/drivers/scsi/hisi_sas/hisi_sas_main.c +++ b/drivers/scsi/hisi_sas/hisi_sas_main.c @@ -2323,6 +2323,7 @@ static struct Scsi_Host *hisi_sas_shost_alloc(struct platform_device *pdev, struct Scsi_Host *shost; struct hisi_hba *hisi_hba; struct device *dev = &pdev->dev; + int error; shost = scsi_host_alloc(hw->sht, sizeof(*hisi_hba)); if (!shost) { @@ -2343,8 +2344,11 @@ static struct Scsi_Host *hisi_sas_shost_alloc(struct platform_device *pdev, if (hisi_sas_get_fw_info(hisi_hba) < 0) goto err_out; - if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)) && - dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32))) { + error = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); + if (error) + error = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); + + if (error) { dev_err(dev, "No usable DMA addressing method\n"); goto err_out; } diff --git a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c index c92b3822c408..e0570fd8466e 100644 --- a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c +++ b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c @@ -2447,10 +2447,12 @@ hisi_sas_v3_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (rc) goto err_out_disable_device; - if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) || - dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32))) { + rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (rc) + rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (rc) { dev_err(dev, "No usable DMA addressing method\n"); - rc = -EIO; + rc = -ENODEV; goto err_out_regions; } diff --git a/drivers/scsi/hptiop.c b/drivers/scsi/hptiop.c index 3eedfd4f8f57..251c084a6ff0 100644 --- a/drivers/scsi/hptiop.c +++ b/drivers/scsi/hptiop.c @@ -1292,6 +1292,7 @@ static int hptiop_probe(struct pci_dev *pcidev, const struct pci_device_id *id) dma_addr_t start_phy; void *start_virt; u32 offset, i, req_size; + int rc; dprintk("hptiop_probe(%p)\n", pcidev); @@ -1308,9 +1309,12 @@ static int hptiop_probe(struct pci_dev *pcidev, const struct pci_device_id *id) /* Enable 64bit DMA if possible */ iop_ops = (struct hptiop_adapter_ops *)id->driver_data; - if (dma_set_mask(&pcidev->dev, - DMA_BIT_MASK(iop_ops->hw_dma_bit_mask)) || - dma_set_mask(&pcidev->dev, DMA_BIT_MASK(32))) { + rc = dma_set_mask(&pcidev->dev, + DMA_BIT_MASK(iop_ops->hw_dma_bit_mask)); + if (rc) + rc = dma_set_mask(&pcidev->dev, DMA_BIT_MASK(32)); + + if (rc) { printk(KERN_ERR "hptiop: fail to set dma_mask\n"); goto disable_pci_device; } diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c index bede11e16349..e1129260ed18 100644 --- a/drivers/scsi/lpfc/lpfc_init.c +++ b/drivers/scsi/lpfc/lpfc_init.c @@ -7361,15 +7361,18 @@ lpfc_sli_pci_mem_setup(struct lpfc_hba *phba) unsigned long bar0map_len, bar2map_len; int i, hbq_count; void *ptr; - int error = -ENODEV; + int error; if (!pdev) - return error; + return -ENODEV; /* Set the device DMA mask size */ - if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) || - dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32))) + error = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (error) + error = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (error) return error; + error = -ENODEV; /* Get the bus address of Bar0 and Bar2 and the number of bytes * required by each mapping. @@ -9742,11 +9745,13 @@ lpfc_sli4_pci_mem_setup(struct lpfc_hba *phba) uint32_t if_type; if (!pdev) - return error; + return -ENODEV; /* Set the device DMA mask size */ - if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) || - dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32))) + error = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (error) + error = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (error) return error; /* diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index f8d51c3d5582..a6828391d6b3 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -2598,7 +2598,6 @@ void scsi_device_resume(struct scsi_device *sdev) * device deleted during suspend) */ mutex_lock(&sdev->state_mutex); - WARN_ON_ONCE(!sdev->quiesced_by); sdev->quiesced_by = NULL; blk_clear_pm_only(sdev->request_queue); if (sdev->sdev_state == SDEV_QUIESCE) diff --git a/drivers/staging/rtl8723bs/os_dep/osdep_service.c b/drivers/staging/rtl8723bs/os_dep/osdep_service.c index e14d7cc411c9..73b87da15eb2 100644 --- a/drivers/staging/rtl8723bs/os_dep/osdep_service.c +++ b/drivers/staging/rtl8723bs/os_dep/osdep_service.c @@ -137,7 +137,7 @@ static int isFileReadable(char *path) ret = PTR_ERR(fp); } else { - oldfs = get_fs(); set_fs(get_ds()); + oldfs = get_fs(); set_fs(KERNEL_DS); if (1!=readFile(fp, &buf, 1)) ret = -EINVAL; @@ -165,7 +165,7 @@ static int retriveFromFile(char *path, u8 *buf, u32 sz) if (0 == (ret =openFile(&fp, path, O_RDONLY, 0))) { DBG_871X("%s openFile path:%s fp =%p\n", __func__, path , fp); - oldfs = get_fs(); set_fs(get_ds()); + oldfs = get_fs(); set_fs(KERNEL_DS); ret =readFile(fp, buf, sz); set_fs(oldfs); closeFile(fp); diff --git a/drivers/tee/optee/core.c b/drivers/tee/optee/core.c index e5efce3c08e2..947f9b28de9e 100644 --- a/drivers/tee/optee/core.c +++ b/drivers/tee/optee/core.c @@ -699,8 +699,10 @@ static int __init optee_driver_init(void) return -ENODEV; np = of_find_matching_node(fw_np, optee_match); - if (!np || !of_device_is_available(np)) + if (!np || !of_device_is_available(np)) { + of_node_put(np); return -ENODEV; + } optee = optee_probe(np); of_node_put(np); diff --git a/drivers/vfio/mdev/mdev_core.c b/drivers/vfio/mdev/mdev_core.c index 0212f0ee8aea..b96fedc77ee5 100644 --- a/drivers/vfio/mdev/mdev_core.c +++ b/drivers/vfio/mdev/mdev_core.c @@ -60,9 +60,9 @@ struct mdev_device *mdev_from_dev(struct device *dev) } EXPORT_SYMBOL(mdev_from_dev); -uuid_le mdev_uuid(struct mdev_device *mdev) +const guid_t *mdev_uuid(struct mdev_device *mdev) { - return mdev->uuid; + return &mdev->uuid; } EXPORT_SYMBOL(mdev_uuid); @@ -88,8 +88,7 @@ static void mdev_release_parent(struct kref *kref) put_device(dev); } -static -inline struct mdev_parent *mdev_get_parent(struct mdev_parent *parent) +static inline struct mdev_parent *mdev_get_parent(struct mdev_parent *parent) { if (parent) kref_get(&parent->ref); @@ -276,7 +275,8 @@ static void mdev_device_release(struct device *dev) kfree(mdev); } -int mdev_device_create(struct kobject *kobj, struct device *dev, uuid_le uuid) +int mdev_device_create(struct kobject *kobj, + struct device *dev, const guid_t *uuid) { int ret; struct mdev_device *mdev, *tmp; @@ -291,7 +291,7 @@ int mdev_device_create(struct kobject *kobj, struct device *dev, uuid_le uuid) /* Check for duplicate */ list_for_each_entry(tmp, &mdev_list, next) { - if (!uuid_le_cmp(tmp->uuid, uuid)) { + if (guid_equal(&tmp->uuid, uuid)) { mutex_unlock(&mdev_list_lock); ret = -EEXIST; goto mdev_fail; @@ -305,7 +305,7 @@ int mdev_device_create(struct kobject *kobj, struct device *dev, uuid_le uuid) goto mdev_fail; } - memcpy(&mdev->uuid, &uuid, sizeof(uuid_le)); + guid_copy(&mdev->uuid, uuid); list_add(&mdev->next, &mdev_list); mutex_unlock(&mdev_list_lock); @@ -315,7 +315,7 @@ int mdev_device_create(struct kobject *kobj, struct device *dev, uuid_le uuid) mdev->dev.parent = dev; mdev->dev.bus = &mdev_bus_type; mdev->dev.release = mdev_device_release; - dev_set_name(&mdev->dev, "%pUl", uuid.b); + dev_set_name(&mdev->dev, "%pUl", uuid); ret = device_register(&mdev->dev); if (ret) { diff --git a/drivers/vfio/mdev/mdev_private.h b/drivers/vfio/mdev/mdev_private.h index b5819b7d7ef7..379758c52b1b 100644 --- a/drivers/vfio/mdev/mdev_private.h +++ b/drivers/vfio/mdev/mdev_private.h @@ -28,7 +28,7 @@ struct mdev_parent { struct mdev_device { struct device dev; struct mdev_parent *parent; - uuid_le uuid; + guid_t uuid; void *driver_data; struct kref ref; struct list_head next; @@ -58,7 +58,8 @@ void parent_remove_sysfs_files(struct mdev_parent *parent); int mdev_create_sysfs_files(struct device *dev, struct mdev_type *type); void mdev_remove_sysfs_files(struct device *dev, struct mdev_type *type); -int mdev_device_create(struct kobject *kobj, struct device *dev, uuid_le uuid); +int mdev_device_create(struct kobject *kobj, + struct device *dev, const guid_t *uuid); int mdev_device_remove(struct device *dev, bool force_remove); #endif /* MDEV_PRIVATE_H */ diff --git a/drivers/vfio/mdev/mdev_sysfs.c b/drivers/vfio/mdev/mdev_sysfs.c index ce5dd219f2c8..5193a0e0ce5a 100644 --- a/drivers/vfio/mdev/mdev_sysfs.c +++ b/drivers/vfio/mdev/mdev_sysfs.c @@ -55,7 +55,7 @@ static ssize_t create_store(struct kobject *kobj, struct device *dev, const char *buf, size_t count) { char *str; - uuid_le uuid; + guid_t uuid; int ret; if ((count < UUID_STRING_LEN) || (count > UUID_STRING_LEN + 1)) @@ -65,12 +65,12 @@ static ssize_t create_store(struct kobject *kobj, struct device *dev, if (!str) return -ENOMEM; - ret = uuid_le_to_bin(str, &uuid); + ret = guid_parse(str, &uuid); kfree(str); if (ret) return ret; - ret = mdev_device_create(kobj, dev, uuid); + ret = mdev_device_create(kobj, dev, &uuid); if (ret) return ret; diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c index ff60bd1ea587..a25659b5a5d1 100644 --- a/drivers/vfio/pci/vfio_pci.c +++ b/drivers/vfio/pci/vfio_pci.c @@ -209,6 +209,57 @@ static bool vfio_pci_nointx(struct pci_dev *pdev) return false; } +static void vfio_pci_probe_power_state(struct vfio_pci_device *vdev) +{ + struct pci_dev *pdev = vdev->pdev; + u16 pmcsr; + + if (!pdev->pm_cap) + return; + + pci_read_config_word(pdev, pdev->pm_cap + PCI_PM_CTRL, &pmcsr); + + vdev->needs_pm_restore = !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET); +} + +/* + * pci_set_power_state() wrapper handling devices which perform a soft reset on + * D3->D0 transition. Save state prior to D0/1/2->D3, stash it on the vdev, + * restore when returned to D0. Saved separately from pci_saved_state for use + * by PM capability emulation and separately from pci_dev internal saved state + * to avoid it being overwritten and consumed around other resets. + */ +int vfio_pci_set_power_state(struct vfio_pci_device *vdev, pci_power_t state) +{ + struct pci_dev *pdev = vdev->pdev; + bool needs_restore = false, needs_save = false; + int ret; + + if (vdev->needs_pm_restore) { + if (pdev->current_state < PCI_D3hot && state >= PCI_D3hot) { + pci_save_state(pdev); + needs_save = true; + } + + if (pdev->current_state >= PCI_D3hot && state <= PCI_D0) + needs_restore = true; + } + + ret = pci_set_power_state(pdev, state); + + if (!ret) { + /* D3 might be unsupported via quirk, skip unless in D3 */ + if (needs_save && pdev->current_state >= PCI_D3hot) { + vdev->pm_save = pci_store_saved_state(pdev); + } else if (needs_restore) { + pci_load_and_free_saved_state(pdev, &vdev->pm_save); + pci_restore_state(pdev); + } + } + + return ret; +} + static int vfio_pci_enable(struct vfio_pci_device *vdev) { struct pci_dev *pdev = vdev->pdev; @@ -216,7 +267,7 @@ static int vfio_pci_enable(struct vfio_pci_device *vdev) u16 cmd; u8 msix_pos; - pci_set_power_state(pdev, PCI_D0); + vfio_pci_set_power_state(vdev, PCI_D0); /* Don't allow our initial saved state to include busmaster */ pci_clear_master(pdev); @@ -407,7 +458,7 @@ out: vfio_pci_try_bus_reset(vdev); if (!disable_idle_d3) - pci_set_power_state(pdev, PCI_D3hot); + vfio_pci_set_power_state(vdev, PCI_D3hot); } static void vfio_pci_release(void *device_data) @@ -708,6 +759,7 @@ static long vfio_pci_ioctl(void *device_data, { void __iomem *io; size_t size; + u16 orig_cmd; info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); info.flags = 0; @@ -723,15 +775,23 @@ static long vfio_pci_ioctl(void *device_data, break; } - /* Is it really there? */ + /* + * Is it really there? Enable memory decode for + * implicit access in pci_map_rom(). + */ + pci_read_config_word(pdev, PCI_COMMAND, &orig_cmd); + pci_write_config_word(pdev, PCI_COMMAND, + orig_cmd | PCI_COMMAND_MEMORY); + io = pci_map_rom(pdev, &size); - if (!io || !size) { + if (io) { + info.flags = VFIO_REGION_INFO_FLAG_READ; + pci_unmap_rom(pdev, io); + } else { info.size = 0; - break; } - pci_unmap_rom(pdev, io); - info.flags = VFIO_REGION_INFO_FLAG_READ; + pci_write_config_word(pdev, PCI_COMMAND, orig_cmd); break; } case VFIO_PCI_VGA_REGION_INDEX: @@ -1286,6 +1346,8 @@ static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) vfio_pci_set_vga_decode(vdev, false)); } + vfio_pci_probe_power_state(vdev); + if (!disable_idle_d3) { /* * pci-core sets the device power state to an unknown value at @@ -1296,8 +1358,8 @@ static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) * be able to get to D3. Therefore first do a D0 transition * before going to D3. */ - pci_set_power_state(pdev, PCI_D0); - pci_set_power_state(pdev, PCI_D3hot); + vfio_pci_set_power_state(vdev, PCI_D0); + vfio_pci_set_power_state(vdev, PCI_D3hot); } return ret; @@ -1316,6 +1378,11 @@ static void vfio_pci_remove(struct pci_dev *pdev) vfio_iommu_group_put(pdev->dev.iommu_group, &pdev->dev); kfree(vdev->region); mutex_destroy(&vdev->ioeventfds_lock); + + if (!disable_idle_d3) + vfio_pci_set_power_state(vdev, PCI_D0); + + kfree(vdev->pm_save); kfree(vdev); if (vfio_pci_is_vga(pdev)) { @@ -1324,9 +1391,6 @@ static void vfio_pci_remove(struct pci_dev *pdev) VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM | VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM); } - - if (!disable_idle_d3) - pci_set_power_state(pdev, PCI_D0); } static pci_ers_result_t vfio_pci_aer_err_detected(struct pci_dev *pdev, @@ -1551,7 +1615,7 @@ put_devs: tmp->needs_reset = false; if (tmp != vdev && !disable_idle_d3) - pci_set_power_state(tmp->pdev, PCI_D3hot); + vfio_pci_set_power_state(tmp, PCI_D3hot); } vfio_device_put(devs.devices[i]); diff --git a/drivers/vfio/pci/vfio_pci_config.c b/drivers/vfio/pci/vfio_pci_config.c index 423ea1f98441..e82b51114687 100644 --- a/drivers/vfio/pci/vfio_pci_config.c +++ b/drivers/vfio/pci/vfio_pci_config.c @@ -691,7 +691,7 @@ static int vfio_pm_config_write(struct vfio_pci_device *vdev, int pos, break; } - pci_set_power_state(vdev->pdev, state); + vfio_pci_set_power_state(vdev, state); } return count; diff --git a/drivers/vfio/pci/vfio_pci_private.h b/drivers/vfio/pci/vfio_pci_private.h index 8c0009f00818..1812cf22fc4f 100644 --- a/drivers/vfio/pci/vfio_pci_private.h +++ b/drivers/vfio/pci/vfio_pci_private.h @@ -114,7 +114,9 @@ struct vfio_pci_device { bool has_vga; bool needs_reset; bool nointx; + bool needs_pm_restore; struct pci_saved_state *pci_saved_state; + struct pci_saved_state *pm_save; struct vfio_pci_reflck *reflck; int refcnt; int ioeventfds_nr; @@ -161,6 +163,10 @@ extern int vfio_pci_register_dev_region(struct vfio_pci_device *vdev, unsigned int type, unsigned int subtype, const struct vfio_pci_regops *ops, size_t size, u32 flags, void *data); + +extern int vfio_pci_set_power_state(struct vfio_pci_device *vdev, + pci_power_t state); + #ifdef CONFIG_VFIO_PCI_IGD extern int vfio_pci_igd_init(struct vfio_pci_device *vdev); #else diff --git a/drivers/vfio/platform/reset/Makefile b/drivers/vfio/platform/reset/Makefile index 57abd4f0ac5b..7294c5ea122e 100644 --- a/drivers/vfio/platform/reset/Makefile +++ b/drivers/vfio/platform/reset/Makefile @@ -2,8 +2,6 @@ vfio-platform-calxedaxgmac-y := vfio_platform_calxedaxgmac.o vfio-platform-amdxgbe-y := vfio_platform_amdxgbe.o -ccflags-y += -Idrivers/vfio/platform - obj-$(CONFIG_VFIO_PLATFORM_CALXEDAXGMAC_RESET) += vfio-platform-calxedaxgmac.o obj-$(CONFIG_VFIO_PLATFORM_AMDXGBE_RESET) += vfio-platform-amdxgbe.o obj-$(CONFIG_VFIO_PLATFORM_BCMFLEXRM_RESET) += vfio_platform_bcmflexrm.o diff --git a/drivers/vfio/platform/reset/vfio_platform_amdxgbe.c b/drivers/vfio/platform/reset/vfio_platform_amdxgbe.c index bcd419cfd79c..3ddb2704221d 100644 --- a/drivers/vfio/platform/reset/vfio_platform_amdxgbe.c +++ b/drivers/vfio/platform/reset/vfio_platform_amdxgbe.c @@ -25,7 +25,7 @@ #include <uapi/linux/mdio.h> #include <linux/delay.h> -#include "vfio_platform_private.h" +#include "../vfio_platform_private.h" #define DMA_MR 0x3000 #define MAC_VR 0x0110 diff --git a/drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c b/drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c index d45c3be71198..16165a62b86d 100644 --- a/drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c +++ b/drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c @@ -23,7 +23,7 @@ #include <linux/kernel.h> #include <linux/module.h> -#include "vfio_platform_private.h" +#include "../vfio_platform_private.h" /* FlexRM configuration */ #define RING_REGS_SIZE 0x10000 diff --git a/drivers/vfio/platform/reset/vfio_platform_calxedaxgmac.c b/drivers/vfio/platform/reset/vfio_platform_calxedaxgmac.c index 49e5df6e8f29..e0356de5df54 100644 --- a/drivers/vfio/platform/reset/vfio_platform_calxedaxgmac.c +++ b/drivers/vfio/platform/reset/vfio_platform_calxedaxgmac.c @@ -24,7 +24,7 @@ #include <linux/init.h> #include <linux/io.h> -#include "vfio_platform_private.h" +#include "../vfio_platform_private.h" #define DRIVER_VERSION "0.1" #define DRIVER_AUTHOR "Eric Auger <eric.auger@linaro.org>" diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c index 64833879f75d..a3030cdf3c18 100644 --- a/drivers/vfio/vfio.c +++ b/drivers/vfio/vfio.c @@ -2219,12 +2219,12 @@ static int __init vfio_init(void) vfio.class->devnode = vfio_devnode; - ret = alloc_chrdev_region(&vfio.group_devt, 0, MINORMASK, "vfio"); + ret = alloc_chrdev_region(&vfio.group_devt, 0, MINORMASK + 1, "vfio"); if (ret) goto err_alloc_chrdev; cdev_init(&vfio.group_cdev, &vfio_group_fops); - ret = cdev_add(&vfio.group_cdev, vfio.group_devt, MINORMASK); + ret = cdev_add(&vfio.group_cdev, vfio.group_devt, MINORMASK + 1); if (ret) goto err_cdev_add; @@ -2236,7 +2236,7 @@ static int __init vfio_init(void) return 0; err_cdev_add: - unregister_chrdev_region(vfio.group_devt, MINORMASK); + unregister_chrdev_region(vfio.group_devt, MINORMASK + 1); err_alloc_chrdev: class_destroy(vfio.class); vfio.class = NULL; @@ -2254,7 +2254,7 @@ static void __exit vfio_cleanup(void) #endif idr_destroy(&vfio.group_idr); cdev_del(&vfio.group_cdev); - unregister_chrdev_region(vfio.group_devt, MINORMASK); + unregister_chrdev_region(vfio.group_devt, MINORMASK + 1); class_destroy(vfio.class); vfio.class = NULL; misc_deregister(&vfio_dev); diff --git a/drivers/vfio/vfio_iommu_spapr_tce.c b/drivers/vfio/vfio_iommu_spapr_tce.c index c424913324e3..8dbb270998f4 100644 --- a/drivers/vfio/vfio_iommu_spapr_tce.c +++ b/drivers/vfio/vfio_iommu_spapr_tce.c @@ -1235,7 +1235,8 @@ static void tce_iommu_release_ownership_ddw(struct tce_container *container, } for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) - table_group->ops->unset_window(table_group, i); + if (container->tables[i]) + table_group->ops->unset_window(table_group, i); table_group->ops->release_ownership(table_group); } diff --git a/fs/afs/cell.c b/fs/afs/cell.c index cf445dbd5f2e..9de46116c749 100644 --- a/fs/afs/cell.c +++ b/fs/afs/cell.c @@ -173,6 +173,7 @@ static struct afs_cell *afs_alloc_cell(struct afs_net *net, rcu_assign_pointer(cell->vl_servers, vllist); cell->dns_expiry = TIME64_MAX; + __clear_bit(AFS_CELL_FL_NO_LOOKUP_YET, &cell->flags); } else { cell->dns_expiry = ktime_get_real_seconds(); } @@ -167,9 +167,13 @@ struct kioctx { unsigned id; }; +/* + * First field must be the file pointer in all the + * iocb unions! See also 'struct kiocb' in <linux/fs.h> + */ struct fsync_iocb { - struct work_struct work; struct file *file; + struct work_struct work; bool datasync; }; @@ -183,8 +187,15 @@ struct poll_iocb { struct work_struct work; }; +/* + * NOTE! Each of the iocb union members has the file pointer + * as the first entry in their struct definition. So you can + * access the file pointer through any of the sub-structs, + * or directly as just 'ki_filp' in this struct. + */ struct aio_kiocb { union { + struct file *ki_filp; struct kiocb rw; struct fsync_iocb fsync; struct poll_iocb poll; @@ -1060,6 +1071,8 @@ static inline void iocb_put(struct aio_kiocb *iocb) { if (refcount_read(&iocb->ki_refcnt) == 0 || refcount_dec_and_test(&iocb->ki_refcnt)) { + if (iocb->ki_filp) + fput(iocb->ki_filp); percpu_ref_put(&iocb->ki_ctx->reqs); kmem_cache_free(kiocb_cachep, iocb); } @@ -1424,7 +1437,6 @@ static void aio_complete_rw(struct kiocb *kiocb, long res, long res2) file_end_write(kiocb->ki_filp); } - fput(kiocb->ki_filp); aio_complete(iocb, res, res2); } @@ -1432,9 +1444,6 @@ static int aio_prep_rw(struct kiocb *req, const struct iocb *iocb) { int ret; - req->ki_filp = fget(iocb->aio_fildes); - if (unlikely(!req->ki_filp)) - return -EBADF; req->ki_complete = aio_complete_rw; req->private = NULL; req->ki_pos = iocb->aio_offset; @@ -1451,7 +1460,7 @@ static int aio_prep_rw(struct kiocb *req, const struct iocb *iocb) ret = ioprio_check_cap(iocb->aio_reqprio); if (ret) { pr_debug("aio ioprio check cap error: %d\n", ret); - goto out_fput; + return ret; } req->ki_ioprio = iocb->aio_reqprio; @@ -1460,14 +1469,10 @@ static int aio_prep_rw(struct kiocb *req, const struct iocb *iocb) ret = kiocb_set_rw_flags(req, iocb->aio_rw_flags); if (unlikely(ret)) - goto out_fput; + return ret; req->ki_flags &= ~IOCB_HIPRI; /* no one is going to poll for this I/O */ return 0; - -out_fput: - fput(req->ki_filp); - return ret; } static int aio_setup_rw(int rw, const struct iocb *iocb, struct iovec **iovec, @@ -1521,24 +1526,19 @@ static ssize_t aio_read(struct kiocb *req, const struct iocb *iocb, if (ret) return ret; file = req->ki_filp; - - ret = -EBADF; if (unlikely(!(file->f_mode & FMODE_READ))) - goto out_fput; + return -EBADF; ret = -EINVAL; if (unlikely(!file->f_op->read_iter)) - goto out_fput; + return -EINVAL; ret = aio_setup_rw(READ, iocb, &iovec, vectored, compat, &iter); if (ret) - goto out_fput; + return ret; ret = rw_verify_area(READ, file, &req->ki_pos, iov_iter_count(&iter)); if (!ret) aio_rw_done(req, call_read_iter(file, req, &iter)); kfree(iovec); -out_fput: - if (unlikely(ret)) - fput(file); return ret; } @@ -1555,16 +1555,14 @@ static ssize_t aio_write(struct kiocb *req, const struct iocb *iocb, return ret; file = req->ki_filp; - ret = -EBADF; if (unlikely(!(file->f_mode & FMODE_WRITE))) - goto out_fput; - ret = -EINVAL; + return -EBADF; if (unlikely(!file->f_op->write_iter)) - goto out_fput; + return -EINVAL; ret = aio_setup_rw(WRITE, iocb, &iovec, vectored, compat, &iter); if (ret) - goto out_fput; + return ret; ret = rw_verify_area(WRITE, file, &req->ki_pos, iov_iter_count(&iter)); if (!ret) { /* @@ -1582,9 +1580,6 @@ static ssize_t aio_write(struct kiocb *req, const struct iocb *iocb, aio_rw_done(req, call_write_iter(file, req, &iter)); } kfree(iovec); -out_fput: - if (unlikely(ret)) - fput(file); return ret; } @@ -1594,7 +1589,6 @@ static void aio_fsync_work(struct work_struct *work) int ret; ret = vfs_fsync(req->file, req->datasync); - fput(req->file); aio_complete(container_of(req, struct aio_kiocb, fsync), ret, 0); } @@ -1605,13 +1599,8 @@ static int aio_fsync(struct fsync_iocb *req, const struct iocb *iocb, iocb->aio_rw_flags)) return -EINVAL; - req->file = fget(iocb->aio_fildes); - if (unlikely(!req->file)) - return -EBADF; - if (unlikely(!req->file->f_op->fsync)) { - fput(req->file); + if (unlikely(!req->file->f_op->fsync)) return -EINVAL; - } req->datasync = datasync; INIT_WORK(&req->work, aio_fsync_work); @@ -1621,10 +1610,7 @@ static int aio_fsync(struct fsync_iocb *req, const struct iocb *iocb, static inline void aio_poll_complete(struct aio_kiocb *iocb, __poll_t mask) { - struct file *file = iocb->poll.file; - aio_complete(iocb, mangle_poll(mask), 0); - fput(file); } static void aio_poll_complete_work(struct work_struct *work) @@ -1680,6 +1666,7 @@ static int aio_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, struct poll_iocb *req = container_of(wait, struct poll_iocb, wait); struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll); __poll_t mask = key_to_poll(key); + unsigned long flags; req->woken = true; @@ -1688,10 +1675,15 @@ static int aio_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, if (!(mask & req->events)) return 0; - /* try to complete the iocb inline if we can: */ - if (spin_trylock(&iocb->ki_ctx->ctx_lock)) { + /* + * Try to complete the iocb inline if we can. Use + * irqsave/irqrestore because not all filesystems (e.g. fuse) + * call this function with IRQs disabled and because IRQs + * have to be disabled before ctx_lock is obtained. + */ + if (spin_trylock_irqsave(&iocb->ki_ctx->ctx_lock, flags)) { list_del(&iocb->ki_list); - spin_unlock(&iocb->ki_ctx->ctx_lock); + spin_unlock_irqrestore(&iocb->ki_ctx->ctx_lock, flags); list_del_init(&req->wait.entry); aio_poll_complete(iocb, mask); @@ -1743,9 +1735,6 @@ static ssize_t aio_poll(struct aio_kiocb *aiocb, const struct iocb *iocb) INIT_WORK(&req->work, aio_poll_complete_work); req->events = demangle_poll(iocb->aio_buf) | EPOLLERR | EPOLLHUP; - req->file = fget(iocb->aio_fildes); - if (unlikely(!req->file)) - return -EBADF; req->head = NULL; req->woken = false; @@ -1788,10 +1777,8 @@ static ssize_t aio_poll(struct aio_kiocb *aiocb, const struct iocb *iocb) spin_unlock_irq(&ctx->ctx_lock); out: - if (unlikely(apt.error)) { - fput(req->file); + if (unlikely(apt.error)) return apt.error; - } if (mask) aio_poll_complete(aiocb, mask); @@ -1829,6 +1816,11 @@ static int __io_submit_one(struct kioctx *ctx, const struct iocb *iocb, if (unlikely(!req)) goto out_put_reqs_available; + req->ki_filp = fget(iocb->aio_fildes); + ret = -EBADF; + if (unlikely(!req->ki_filp)) + goto out_put_req; + if (iocb->aio_flags & IOCB_FLAG_RESFD) { /* * If the IOCB_FLAG_RESFD flag of aio_flags is set, get an diff --git a/fs/exec.c b/fs/exec.c index fb72d36f7823..bcf383730bea 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -932,7 +932,7 @@ int kernel_read_file(struct file *file, void **buf, loff_t *size, bytes = kernel_read(file, *buf + pos, i_size - pos, &pos); if (bytes < 0) { ret = bytes; - goto out; + goto out_free; } if (bytes == 0) diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 32920a10100e..a7fa037b876b 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -859,6 +859,18 @@ static int hugetlbfs_migrate_page(struct address_space *mapping, rc = migrate_huge_page_move_mapping(mapping, newpage, page); if (rc != MIGRATEPAGE_SUCCESS) return rc; + + /* + * page_private is subpool pointer in hugetlb pages. Transfer to + * new page. PagePrivate is not associated with page_private for + * hugetlb pages and can not be set here as only page_huge_active + * pages can be migrated. + */ + if (page_private(page)) { + set_page_private(newpage, page_private(page)); + set_page_private(page, 0); + } + if (mode != MIGRATE_SYNC_NO_COPY) migrate_page_copy(newpage, page); else diff --git a/fs/kernfs/mount.c b/fs/kernfs/mount.c index fdf527b6d79c..d71c9405874a 100644 --- a/fs/kernfs/mount.c +++ b/fs/kernfs/mount.c @@ -196,8 +196,10 @@ struct dentry *kernfs_node_dentry(struct kernfs_node *kn, return dentry; knparent = find_next_ancestor(kn, NULL); - if (WARN_ON(!knparent)) + if (WARN_ON(!knparent)) { + dput(dentry); return ERR_PTR(-EINVAL); + } do { struct dentry *dtmp; @@ -206,8 +208,10 @@ struct dentry *kernfs_node_dentry(struct kernfs_node *kn, if (kn == knparent) return dentry; kntmp = find_next_ancestor(kn, knparent); - if (WARN_ON(!kntmp)) + if (WARN_ON(!kntmp)) { + dput(dentry); return ERR_PTR(-EINVAL); + } dtmp = lookup_one_len_unlocked(kntmp->name, dentry, strlen(kntmp->name)); dput(dentry); diff --git a/fs/namespace.c b/fs/namespace.c index a677b59efd74..c4e83d94840c 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -2698,7 +2698,6 @@ static long exact_copy_from_user(void *to, const void __user * from, if (!access_ok(from, n)) return n; - current->kernel_uaccess_faults_ok++; while (n) { if (__get_user(c, f)) { memset(t, 0, n); @@ -2708,7 +2707,6 @@ static long exact_copy_from_user(void *to, const void __user * from, f++; n--; } - current->kernel_uaccess_faults_ok--; return n; } @@ -2746,7 +2744,7 @@ void *copy_mount_options(const void __user * data) char *copy_mount_string(const void __user *data) { - return data ? strndup_user(data, PAGE_SIZE) : NULL; + return data ? strndup_user(data, PATH_MAX) : NULL; } /* diff --git a/fs/orangefs/file.c b/fs/orangefs/file.c index a5a2fe76568f..b094d3d79354 100644 --- a/fs/orangefs/file.c +++ b/fs/orangefs/file.c @@ -398,8 +398,6 @@ static ssize_t orangefs_file_read_iter(struct kiocb *iocb, struct iov_iter *iter loff_t pos = iocb->ki_pos; ssize_t rc = 0; - BUG_ON(iocb->private); - gossip_debug(GOSSIP_FILE_DEBUG, "orangefs_file_read_iter\n"); orangefs_stats.reads++; @@ -416,8 +414,6 @@ static ssize_t orangefs_file_write_iter(struct kiocb *iocb, struct iov_iter *ite loff_t pos; ssize_t rc; - BUG_ON(iocb->private); - gossip_debug(GOSSIP_FILE_DEBUG, "orangefs_file_write_iter\n"); inode_lock(file->f_mapping->host); diff --git a/fs/read_write.c b/fs/read_write.c index ff3c5e6f87cf..30df848b7451 100644 --- a/fs/read_write.c +++ b/fs/read_write.c @@ -426,7 +426,7 @@ ssize_t kernel_read(struct file *file, void *buf, size_t count, loff_t *pos) ssize_t result; old_fs = get_fs(); - set_fs(get_ds()); + set_fs(KERNEL_DS); /* The cast to a user pointer is valid due to the set_fs() */ result = vfs_read(file, (void __user *)buf, count, pos); set_fs(old_fs); @@ -499,7 +499,7 @@ ssize_t __kernel_write(struct file *file, const void *buf, size_t count, loff_t return -EINVAL; old_fs = get_fs(); - set_fs(get_ds()); + set_fs(KERNEL_DS); p = (__force const char __user *)buf; if (count > MAX_RW_COUNT) count = MAX_RW_COUNT; @@ -521,7 +521,7 @@ ssize_t kernel_write(struct file *file, const void *buf, size_t count, ssize_t res; old_fs = get_fs(); - set_fs(get_ds()); + set_fs(KERNEL_DS); /* The cast to a user pointer is valid due to the set_fs() */ res = vfs_write(file, (__force const char __user *)buf, count, pos); set_fs(old_fs); diff --git a/fs/splice.c b/fs/splice.c index de2ede048473..6489fb9436e4 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -357,7 +357,7 @@ static ssize_t kernel_readv(struct file *file, const struct kvec *vec, ssize_t res; old_fs = get_fs(); - set_fs(get_ds()); + set_fs(KERNEL_DS); /* The cast to a user pointer is valid due to the set_fs() */ res = vfs_readv(file, (const struct iovec __user *)vec, vlen, &pos, 0); set_fs(old_fs); @@ -1123,6 +1123,9 @@ static long do_splice(struct file *in, loff_t __user *off_in, if (ipipe == opipe) return -EINVAL; + if ((in->f_flags | out->f_flags) & O_NONBLOCK) + flags |= SPLICE_F_NONBLOCK; + return splice_pipe_to_pipe(ipipe, opipe, len, flags); } @@ -1148,6 +1151,9 @@ static long do_splice(struct file *in, loff_t __user *off_in, if (unlikely(ret < 0)) return ret; + if (in->f_flags & O_NONBLOCK) + flags |= SPLICE_F_NONBLOCK; + file_start_write(out); ret = do_splice_from(ipipe, out, &offset, len, flags); file_end_write(out); @@ -1172,6 +1178,9 @@ static long do_splice(struct file *in, loff_t __user *off_in, offset = in->f_pos; } + if (out->f_flags & O_NONBLOCK) + flags |= SPLICE_F_NONBLOCK; + pipe_lock(opipe); ret = wait_for_space(opipe, flags); if (!ret) @@ -1717,6 +1726,9 @@ static long do_tee(struct file *in, struct file *out, size_t len, * copying the data. */ if (ipipe && opipe && ipipe != opipe) { + if ((in->f_flags | out->f_flags) & O_NONBLOCK) + flags |= SPLICE_F_NONBLOCK; + /* * Keep going, unless we encounter an error. The ipipe/opipe * ordering doesn't really matter. diff --git a/include/asm-generic/uaccess.h b/include/asm-generic/uaccess.h index d82c78a79da5..b3d2241e03f8 100644 --- a/include/asm-generic/uaccess.h +++ b/include/asm-generic/uaccess.h @@ -22,7 +22,6 @@ #endif #ifndef get_fs -#define get_ds() (KERNEL_DS) #define get_fs() (current_thread_info()->addr_limit) static inline void set_fs(mm_segment_t fs) diff --git a/include/linux/fs.h b/include/linux/fs.h index 29d8e2cfed0e..fd423fec8d83 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -304,13 +304,19 @@ enum rw_hint { struct kiocb { struct file *ki_filp; + + /* The 'ki_filp' pointer is shared in a union for aio */ + randomized_struct_fields_start + loff_t ki_pos; void (*ki_complete)(struct kiocb *iocb, long ret, long ret2); void *private; int ki_flags; u16 ki_hint; u16 ki_ioprio; /* See linux/ioprio.h */ -} __randomize_layout; + + randomized_struct_fields_end +}; static inline bool is_sync_kiocb(struct kiocb *kiocb) { diff --git a/include/linux/mdev.h b/include/linux/mdev.h index b6e048e1045f..d7aee90e5da5 100644 --- a/include/linux/mdev.h +++ b/include/linux/mdev.h @@ -120,7 +120,7 @@ struct mdev_driver { extern void *mdev_get_drvdata(struct mdev_device *mdev); extern void mdev_set_drvdata(struct mdev_device *mdev, void *data); -extern uuid_le mdev_uuid(struct mdev_device *mdev); +extern const guid_t *mdev_uuid(struct mdev_device *mdev); extern struct bus_type mdev_bus_type; diff --git a/include/linux/mtd/rawnand.h b/include/linux/mtd/rawnand.h index 33e240acdc6d..b7445a44a814 100644 --- a/include/linux/mtd/rawnand.h +++ b/include/linux/mtd/rawnand.h @@ -16,13 +16,12 @@ #ifndef __LINUX_MTD_RAWNAND_H #define __LINUX_MTD_RAWNAND_H -#include <linux/wait.h> -#include <linux/spinlock.h> #include <linux/mtd/mtd.h> #include <linux/mtd/flashchip.h> #include <linux/mtd/bbm.h> #include <linux/mtd/jedec.h> #include <linux/mtd/onfi.h> +#include <linux/mutex.h> #include <linux/of.h> #include <linux/types.h> @@ -897,25 +896,17 @@ struct nand_controller_ops { /** * struct nand_controller - Structure used to describe a NAND controller * - * @lock: protection lock - * @active: the mtd device which holds the controller currently - * @wq: wait queue to sleep on if a NAND operation is in - * progress used instead of the per chip wait queue - * when a hw controller is available. + * @lock: lock used to serialize accesses to the NAND controller * @ops: NAND controller operations. */ struct nand_controller { - spinlock_t lock; - struct nand_chip *active; - wait_queue_head_t wq; + struct mutex lock; const struct nand_controller_ops *ops; }; static inline void nand_controller_init(struct nand_controller *nfc) { - nfc->active = NULL; - spin_lock_init(&nfc->lock); - init_waitqueue_head(&nfc->wq); + mutex_init(&nfc->lock); } /** @@ -936,7 +927,6 @@ static inline void nand_controller_init(struct nand_controller *nfc) * @waitfunc: hardware specific function for wait on ready. * @block_bad: check if a block is bad, using OOB markers * @block_markbad: mark a block bad - * @erase: erase function * @set_features: set the NAND chip features * @get_features: get the NAND chip features * @chip_delay: chip dependent delay for transferring data from array to read @@ -962,7 +952,6 @@ struct nand_legacy { int (*waitfunc)(struct nand_chip *chip); int (*block_bad)(struct nand_chip *chip, loff_t ofs); int (*block_markbad)(struct nand_chip *chip, loff_t ofs); - int (*erase)(struct nand_chip *chip, int page); int (*set_features)(struct nand_chip *chip, int feature_addr, u8 *subfeature_para); int (*get_features)(struct nand_chip *chip, int feature_addr, @@ -983,7 +972,6 @@ struct nand_legacy { * setting the read-retry mode. Mostly needed for MLC NAND. * @ecc: [BOARDSPECIFIC] ECC control structure * @buf_align: minimum buffer alignment required by a platform - * @state: [INTERN] the current state of the NAND device * @oob_poi: "poison value buffer," used for laying out OOB data * before writing * @page_shift: [INTERN] number of address bits in a page (column @@ -1034,6 +1022,9 @@ struct nand_legacy { * cur_cs < numchips. NAND Controller drivers should not * modify this value, but they're allowed to read it. * @read_retries: [INTERN] the number of read retry modes supported + * @lock: lock protecting the suspended field. Also used to + * serialize accesses to the NAND device. + * @suspended: set to 1 when the device is suspended, 0 when it's not. * @bbt: [INTERN] bad block table pointer * @bbt_td: [REPLACEABLE] bad block table descriptor for flash * lookup. @@ -1088,7 +1079,8 @@ struct nand_chip { int read_retries; - flstate_t state; + struct mutex lock; + unsigned int suspended : 1; uint8_t *oob_poi; struct nand_controller *controller; diff --git a/include/linux/mtd/spi-nor.h b/include/linux/mtd/spi-nor.h index fa2d89e38e40..b3d360b0ee3d 100644 --- a/include/linux/mtd/spi-nor.h +++ b/include/linux/mtd/spi-nor.h @@ -46,9 +46,13 @@ #define SPINOR_OP_READ_1_2_2 0xbb /* Read data bytes (Dual I/O SPI) */ #define SPINOR_OP_READ_1_1_4 0x6b /* Read data bytes (Quad Output SPI) */ #define SPINOR_OP_READ_1_4_4 0xeb /* Read data bytes (Quad I/O SPI) */ +#define SPINOR_OP_READ_1_1_8 0x8b /* Read data bytes (Octal Output SPI) */ +#define SPINOR_OP_READ_1_8_8 0xcb /* Read data bytes (Octal I/O SPI) */ #define SPINOR_OP_PP 0x02 /* Page program (up to 256 bytes) */ #define SPINOR_OP_PP_1_1_4 0x32 /* Quad page program */ #define SPINOR_OP_PP_1_4_4 0x38 /* Quad page program */ +#define SPINOR_OP_PP_1_1_8 0x82 /* Octal page program */ +#define SPINOR_OP_PP_1_8_8 0xc2 /* Octal page program */ #define SPINOR_OP_BE_4K 0x20 /* Erase 4KiB block */ #define SPINOR_OP_BE_4K_PMC 0xd7 /* Erase 4KiB block on PMC chips */ #define SPINOR_OP_BE_32K 0x52 /* Erase 32KiB block */ @@ -69,9 +73,13 @@ #define SPINOR_OP_READ_1_2_2_4B 0xbc /* Read data bytes (Dual I/O SPI) */ #define SPINOR_OP_READ_1_1_4_4B 0x6c /* Read data bytes (Quad Output SPI) */ #define SPINOR_OP_READ_1_4_4_4B 0xec /* Read data bytes (Quad I/O SPI) */ +#define SPINOR_OP_READ_1_1_8_4B 0x7c /* Read data bytes (Octal Output SPI) */ +#define SPINOR_OP_READ_1_8_8_4B 0xcc /* Read data bytes (Octal I/O SPI) */ #define SPINOR_OP_PP_4B 0x12 /* Page program (up to 256 bytes) */ #define SPINOR_OP_PP_1_1_4_4B 0x34 /* Quad page program */ #define SPINOR_OP_PP_1_4_4_4B 0x3e /* Quad page program */ +#define SPINOR_OP_PP_1_1_8_4B 0x84 /* Octal page program */ +#define SPINOR_OP_PP_1_8_8_4B 0x8e /* Octal page program */ #define SPINOR_OP_BE_4K_4B 0x21 /* Erase 4KiB block */ #define SPINOR_OP_BE_32K_4B 0x5c /* Erase 32KiB block */ #define SPINOR_OP_SE_4B 0xdc /* Sector erase (usually 64KiB) */ @@ -458,7 +466,7 @@ struct spi_nor_hwcaps { /* *(Fast) Read capabilities. * MUST be ordered by priority: the higher bit position, the higher priority. - * As a matter of performances, it is relevant to use Octo SPI protocols first, + * As a matter of performances, it is relevant to use Octal SPI protocols first, * then Quad SPI protocols before Dual SPI protocols, Fast Read and lastly * (Slow) Read. */ @@ -479,7 +487,7 @@ struct spi_nor_hwcaps { #define SNOR_HWCAPS_READ_4_4_4 BIT(9) #define SNOR_HWCAPS_READ_1_4_4_DTR BIT(10) -#define SNOR_HWCPAS_READ_OCTO GENMASK(14, 11) +#define SNOR_HWCAPS_READ_OCTAL GENMASK(14, 11) #define SNOR_HWCAPS_READ_1_1_8 BIT(11) #define SNOR_HWCAPS_READ_1_8_8 BIT(12) #define SNOR_HWCAPS_READ_8_8_8 BIT(13) @@ -488,7 +496,7 @@ struct spi_nor_hwcaps { /* * Page Program capabilities. * MUST be ordered by priority: the higher bit position, the higher priority. - * Like (Fast) Read capabilities, Octo/Quad SPI protocols are preferred to the + * Like (Fast) Read capabilities, Octal/Quad SPI protocols are preferred to the * legacy SPI 1-1-1 protocol. * Note that Dual Page Programs are not supported because there is no existing * JEDEC/SFDP standard to define them. Also at this moment no SPI flash memory @@ -502,7 +510,7 @@ struct spi_nor_hwcaps { #define SNOR_HWCAPS_PP_1_4_4 BIT(18) #define SNOR_HWCAPS_PP_4_4_4 BIT(19) -#define SNOR_HWCAPS_PP_OCTO GENMASK(22, 20) +#define SNOR_HWCAPS_PP_OCTAL GENMASK(22, 20) #define SNOR_HWCAPS_PP_1_1_8 BIT(20) #define SNOR_HWCAPS_PP_1_8_8 BIT(21) #define SNOR_HWCAPS_PP_8_8_8 BIT(22) diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 86dbb3e29139..848b54b7ec91 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -3861,7 +3861,7 @@ static inline u32 netif_msg_init(int debug_value, int default_msg_enable_bits) if (debug_value == 0) /* no output */ return 0; /* set low N bits */ - return (1 << debug_value) - 1; + return (1U << debug_value) - 1; } static inline void __netif_tx_lock(struct netdev_queue *txq, int cpu) diff --git a/include/linux/sched.h b/include/linux/sched.h index bba3afb4e9bf..f9b43c989577 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -739,12 +739,6 @@ struct task_struct { unsigned use_memdelay:1; #endif - /* - * May usercopy functions fault on kernel addresses? - * This is not just a single bit because this can potentially nest. - */ - unsigned int kernel_uaccess_faults_ok; - unsigned long atomic_flags; /* Flags requiring atomic access. */ struct restart_block restart_block; diff --git a/include/net/icmp.h b/include/net/icmp.h index 6ac3a5bd0117..e0f709d26dde 100644 --- a/include/net/icmp.h +++ b/include/net/icmp.h @@ -22,6 +22,7 @@ #include <net/inet_sock.h> #include <net/snmp.h> +#include <net/ip.h> struct icmp_err { int errno; @@ -39,7 +40,13 @@ struct net_proto_family; struct sk_buff; struct net; -void icmp_send(struct sk_buff *skb_in, int type, int code, __be32 info); +void __icmp_send(struct sk_buff *skb_in, int type, int code, __be32 info, + const struct ip_options *opt); +static inline void icmp_send(struct sk_buff *skb_in, int type, int code, __be32 info) +{ + __icmp_send(skb_in, type, code, info, &IPCB(skb_in)->opt); +} + int icmp_rcv(struct sk_buff *skb); int icmp_err(struct sk_buff *skb, u32 info); int icmp_init(void); diff --git a/include/net/ip.h b/include/net/ip.h index 8866bfce6121..be3cad9c2e4c 100644 --- a/include/net/ip.h +++ b/include/net/ip.h @@ -667,6 +667,8 @@ static inline int ip_options_echo(struct net *net, struct ip_options *dopt, } void ip_options_fragment(struct sk_buff *skb); +int __ip_options_compile(struct net *net, struct ip_options *opt, + struct sk_buff *skb, __be32 *info); int ip_options_compile(struct net *net, struct ip_options *opt, struct sk_buff *skb); int ip_options_get(struct net *net, struct ip_options_rcu **optp, @@ -716,7 +718,7 @@ extern int sysctl_icmp_msgs_burst; int ip_misc_proc_init(void); #endif -int rtm_getroute_parse_ip_proto(struct nlattr *attr, u8 *ip_proto, +int rtm_getroute_parse_ip_proto(struct nlattr *attr, u8 *ip_proto, u8 family, struct netlink_ext_ack *extack); #endif /* _IP_H */ diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 8577bb7f8be6..84470d1480aa 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -559,12 +559,12 @@ static int map_create(union bpf_attr *attr) err = bpf_map_new_fd(map, f_flags); if (err < 0) { /* failed to allocate fd. - * bpf_map_put() is needed because the above + * bpf_map_put_with_uref() is needed because the above * bpf_map_alloc_id() has published the map * to the userspace and the userspace may * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID. */ - bpf_map_put(map); + bpf_map_put_with_uref(map); return err; } @@ -1986,7 +1986,7 @@ static int bpf_map_get_fd_by_id(const union bpf_attr *attr) fd = bpf_map_new_fd(map, f_flags); if (fd < 0) - bpf_map_put(map); + bpf_map_put_with_uref(map); return fd; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 8f295b790297..5fcce2f4209d 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6920,7 +6920,8 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env) u32 off_reg; aux = &env->insn_aux_data[i + delta]; - if (!aux->alu_state) + if (!aux->alu_state || + aux->alu_state == BPF_ALU_NON_POINTER) continue; isneg = aux->alu_state & BPF_ALU_NEG_VALUE; diff --git a/kernel/cgroup/cgroup-internal.h b/kernel/cgroup/cgroup-internal.h index c950864016e2..c9a35f09e4b9 100644 --- a/kernel/cgroup/cgroup-internal.h +++ b/kernel/cgroup/cgroup-internal.h @@ -198,7 +198,7 @@ int cgroup_path_ns_locked(struct cgroup *cgrp, char *buf, size_t buflen, void cgroup_free_root(struct cgroup_root *root); void init_cgroup_root(struct cgroup_root *root, struct cgroup_sb_opts *opts); -int cgroup_setup_root(struct cgroup_root *root, u16 ss_mask, int ref_flags); +int cgroup_setup_root(struct cgroup_root *root, u16 ss_mask); int rebind_subsystems(struct cgroup_root *dst_root, u16 ss_mask); struct dentry *cgroup_do_mount(struct file_system_type *fs_type, int flags, struct cgroup_root *root, unsigned long magic, diff --git a/kernel/cgroup/cgroup-v1.c b/kernel/cgroup/cgroup-v1.c index 583b969b0c0e..f94a7229974e 100644 --- a/kernel/cgroup/cgroup-v1.c +++ b/kernel/cgroup/cgroup-v1.c @@ -1116,13 +1116,11 @@ struct dentry *cgroup1_mount(struct file_system_type *fs_type, int flags, void *data, unsigned long magic, struct cgroup_namespace *ns) { - struct super_block *pinned_sb = NULL; struct cgroup_sb_opts opts; struct cgroup_root *root; struct cgroup_subsys *ss; struct dentry *dentry; int i, ret; - bool new_root = false; cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp); @@ -1184,29 +1182,6 @@ struct dentry *cgroup1_mount(struct file_system_type *fs_type, int flags, if (root->flags ^ opts.flags) pr_warn("new mount options do not match the existing superblock, will be ignored\n"); - /* - * We want to reuse @root whose lifetime is governed by its - * ->cgrp. Let's check whether @root is alive and keep it - * that way. As cgroup_kill_sb() can happen anytime, we - * want to block it by pinning the sb so that @root doesn't - * get killed before mount is complete. - * - * With the sb pinned, tryget_live can reliably indicate - * whether @root can be reused. If it's being killed, - * drain it. We can use wait_queue for the wait but this - * path is super cold. Let's just sleep a bit and retry. - */ - pinned_sb = kernfs_pin_sb(root->kf_root, NULL); - if (IS_ERR(pinned_sb) || - !percpu_ref_tryget_live(&root->cgrp.self.refcnt)) { - mutex_unlock(&cgroup_mutex); - if (!IS_ERR_OR_NULL(pinned_sb)) - deactivate_super(pinned_sb); - msleep(10); - ret = restart_syscall(); - goto out_free; - } - ret = 0; goto out_unlock; } @@ -1232,15 +1207,20 @@ struct dentry *cgroup1_mount(struct file_system_type *fs_type, int flags, ret = -ENOMEM; goto out_unlock; } - new_root = true; init_cgroup_root(root, &opts); - ret = cgroup_setup_root(root, opts.subsys_mask, PERCPU_REF_INIT_DEAD); + ret = cgroup_setup_root(root, opts.subsys_mask); if (ret) cgroup_free_root(root); out_unlock: + if (!ret && !percpu_ref_tryget_live(&root->cgrp.self.refcnt)) { + mutex_unlock(&cgroup_mutex); + msleep(10); + ret = restart_syscall(); + goto out_free; + } mutex_unlock(&cgroup_mutex); out_free: kfree(opts.release_agent); @@ -1252,25 +1232,13 @@ out_free: dentry = cgroup_do_mount(&cgroup_fs_type, flags, root, CGROUP_SUPER_MAGIC, ns); - /* - * There's a race window after we release cgroup_mutex and before - * allocating a superblock. Make sure a concurrent process won't - * be able to re-use the root during this window by delaying the - * initialization of root refcnt. - */ - if (new_root) { - mutex_lock(&cgroup_mutex); - percpu_ref_reinit(&root->cgrp.self.refcnt); - mutex_unlock(&cgroup_mutex); + if (!IS_ERR(dentry) && percpu_ref_is_dying(&root->cgrp.self.refcnt)) { + struct super_block *sb = dentry->d_sb; + dput(dentry); + deactivate_locked_super(sb); + msleep(10); + dentry = ERR_PTR(restart_syscall()); } - - /* - * If @pinned_sb, we're reusing an existing root and holding an - * extra ref on its sb. Mount is complete. Put the extra ref. - */ - if (pinned_sb) - deactivate_super(pinned_sb); - return dentry; } diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index f31bd61c9466..7fd9f22e406d 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -1927,7 +1927,7 @@ void init_cgroup_root(struct cgroup_root *root, struct cgroup_sb_opts *opts) set_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags); } -int cgroup_setup_root(struct cgroup_root *root, u16 ss_mask, int ref_flags) +int cgroup_setup_root(struct cgroup_root *root, u16 ss_mask) { LIST_HEAD(tmp_links); struct cgroup *root_cgrp = &root->cgrp; @@ -1944,7 +1944,7 @@ int cgroup_setup_root(struct cgroup_root *root, u16 ss_mask, int ref_flags) root_cgrp->ancestor_ids[0] = ret; ret = percpu_ref_init(&root_cgrp->self.refcnt, css_release, - ref_flags, GFP_KERNEL); + 0, GFP_KERNEL); if (ret) goto out; @@ -2033,7 +2033,7 @@ struct dentry *cgroup_do_mount(struct file_system_type *fs_type, int flags, struct cgroup_namespace *ns) { struct dentry *dentry; - bool new_sb; + bool new_sb = false; dentry = kernfs_mount(fs_type, flags, root->kf_root, magic, &new_sb); @@ -2043,6 +2043,7 @@ struct dentry *cgroup_do_mount(struct file_system_type *fs_type, int flags, */ if (!IS_ERR(dentry) && ns != &init_cgroup_ns) { struct dentry *nsdentry; + struct super_block *sb = dentry->d_sb; struct cgroup *cgrp; mutex_lock(&cgroup_mutex); @@ -2053,12 +2054,14 @@ struct dentry *cgroup_do_mount(struct file_system_type *fs_type, int flags, spin_unlock_irq(&css_set_lock); mutex_unlock(&cgroup_mutex); - nsdentry = kernfs_node_dentry(cgrp->kn, dentry->d_sb); + nsdentry = kernfs_node_dentry(cgrp->kn, sb); dput(dentry); + if (IS_ERR(nsdentry)) + deactivate_locked_super(sb); dentry = nsdentry; } - if (IS_ERR(dentry) || !new_sb) + if (!new_sb) cgroup_put(&root->cgrp); return dentry; @@ -2118,18 +2121,16 @@ static void cgroup_kill_sb(struct super_block *sb) struct cgroup_root *root = cgroup_root_from_kf(kf_root); /* - * If @root doesn't have any mounts or children, start killing it. + * If @root doesn't have any children, start killing it. * This prevents new mounts by disabling percpu_ref_tryget_live(). * cgroup_mount() may wait for @root's release. * * And don't kill the default root. */ - if (!list_empty(&root->cgrp.self.children) || - root == &cgrp_dfl_root) - cgroup_put(&root->cgrp); - else + if (list_empty(&root->cgrp.self.children) && root != &cgrp_dfl_root && + !percpu_ref_is_dying(&root->cgrp.self.refcnt)) percpu_ref_kill(&root->cgrp.self.refcnt); - + cgroup_put(&root->cgrp); kernfs_kill_sb(sb); } @@ -5399,7 +5400,7 @@ int __init cgroup_init(void) hash_add(css_set_table, &init_css_set.hlist, css_set_hash(init_css_set.subsys)); - BUG_ON(cgroup_setup_root(&cgrp_dfl_root, 0, 0)); + BUG_ON(cgroup_setup_root(&cgrp_dfl_root, 0)); mutex_unlock(&cgroup_mutex); diff --git a/lib/Kconfig.kasan b/lib/Kconfig.kasan index d8c474b6691e..9737059ec58b 100644 --- a/lib/Kconfig.kasan +++ b/lib/Kconfig.kasan @@ -113,6 +113,28 @@ config KASAN_INLINE endchoice +config KASAN_STACK_ENABLE + bool "Enable stack instrumentation (unsafe)" if CC_IS_CLANG && !COMPILE_TEST + default !(CLANG_VERSION < 90000) + depends on KASAN + help + The LLVM stack address sanitizer has a know problem that + causes excessive stack usage in a lot of functions, see + https://bugs.llvm.org/show_bug.cgi?id=38809 + Disabling asan-stack makes it safe to run kernels build + with clang-8 with KASAN enabled, though it loses some of + the functionality. + This feature is always disabled when compile-testing with clang-8 + or earlier to avoid cluttering the output in stack overflow + warnings, but clang-8 users can still enable it for builds without + CONFIG_COMPILE_TEST. On gcc and later clang versions it is + assumed to always be safe to use and enabled by default. + +config KASAN_STACK + int + default 1 if KASAN_STACK_ENABLE || CC_IS_GCC + default 0 + config KASAN_S390_4_LEVEL_PAGING bool "KASan: use 4-level paging" depends on KASAN && S390 diff --git a/mm/hugetlb.c b/mm/hugetlb.c index afef61656c1e..8dfdffc34a99 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3624,7 +3624,6 @@ retry_avoidcopy: copy_user_huge_page(new_page, old_page, address, vma, pages_per_huge_page(h)); __SetPageUptodate(new_page); - set_page_huge_active(new_page); mmu_notifier_range_init(&range, mm, haddr, haddr + huge_page_size(h)); mmu_notifier_invalidate_range_start(&range); @@ -3645,6 +3644,7 @@ retry_avoidcopy: make_huge_pte(vma, new_page, 1)); page_remove_rmap(old_page, true); hugepage_add_new_anon_rmap(new_page, vma, haddr); + set_page_huge_active(new_page); /* Make the old page be freed below */ new_page = old_page; } @@ -3729,6 +3729,7 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, pte_t new_pte; spinlock_t *ptl; unsigned long haddr = address & huge_page_mask(h); + bool new_page = false; /* * Currently, we are forced to kill the process in the event the @@ -3790,7 +3791,7 @@ retry: } clear_huge_page(page, address, pages_per_huge_page(h)); __SetPageUptodate(page); - set_page_huge_active(page); + new_page = true; if (vma->vm_flags & VM_MAYSHARE) { int err = huge_add_to_page_cache(page, mapping, idx); @@ -3861,6 +3862,15 @@ retry: } spin_unlock(ptl); + + /* + * Only make newly allocated pages active. Existing pages found + * in the pagecache could be !page_huge_active() if they have been + * isolated for migration. + */ + if (new_page) + set_page_huge_active(page); + unlock_page(page); out: return ret; @@ -4095,7 +4105,6 @@ int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, * the set_pte_at() write. */ __SetPageUptodate(page); - set_page_huge_active(page); mapping = dst_vma->vm_file->f_mapping; idx = vma_hugecache_offset(h, dst_vma, dst_addr); @@ -4163,6 +4172,7 @@ int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, update_mmu_cache(dst_vma, dst_addr, dst_pte); spin_unlock(ptl); + set_page_huge_active(page); if (vm_shared) unlock_page(page); ret = 0; diff --git a/mm/maccess.c b/mm/maccess.c index f3416632e5a4..ec00be51a24f 100644 --- a/mm/maccess.c +++ b/mm/maccess.c @@ -30,10 +30,8 @@ long __probe_kernel_read(void *dst, const void *src, size_t size) set_fs(KERNEL_DS); pagefault_disable(); - current->kernel_uaccess_faults_ok++; ret = __copy_from_user_inatomic(dst, (__force const void __user *)src, size); - current->kernel_uaccess_faults_ok--; pagefault_enable(); set_fs(old_fs); @@ -60,9 +58,7 @@ long __probe_kernel_write(void *dst, const void *src, size_t size) set_fs(KERNEL_DS); pagefault_disable(); - current->kernel_uaccess_faults_ok++; ret = __copy_to_user_inatomic((__force void __user *)dst, src, size); - current->kernel_uaccess_faults_ok--; pagefault_enable(); set_fs(old_fs); @@ -98,13 +94,11 @@ long strncpy_from_unsafe(char *dst, const void *unsafe_addr, long count) set_fs(KERNEL_DS); pagefault_disable(); - current->kernel_uaccess_faults_ok++; do { ret = __get_user(*dst++, (const char __user __force *)src++); } while (dst[-1] && ret == 0 && src - unsafe_addr < count); - current->kernel_uaccess_faults_ok--; dst[-1] = '\0'; pagefault_enable(); set_fs(old_fs); diff --git a/mm/migrate.c b/mm/migrate.c index d4fd680be3b0..181f5d2718a9 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -1315,6 +1315,16 @@ static int unmap_and_move_huge_page(new_page_t get_new_page, lock_page(hpage); } + /* + * Check for pages which are in the process of being freed. Without + * page_mapping() set, hugetlbfs specific move page routine will not + * be called and we could leak usage counts for subpools. + */ + if (page_private(hpage) && !page_mapping(hpage)) { + rc = -EBUSY; + goto out_unlock; + } + if (PageAnon(hpage)) anon_vma = page_get_anon_vma(hpage); @@ -1345,6 +1355,7 @@ put_anon: put_new_page = NULL; } +out_unlock: unlock_page(hpage); out: if (rc != -EAGAIN) diff --git a/mm/mmap.c b/mm/mmap.c index f901065c4c64..fc1809b1bed6 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -2426,12 +2426,11 @@ int expand_downwards(struct vm_area_struct *vma, { struct mm_struct *mm = vma->vm_mm; struct vm_area_struct *prev; - int error; + int error = 0; address &= PAGE_MASK; - error = security_mmap_addr(address); - if (error) - return error; + if (address < mmap_min_addr) + return -EPERM; /* Enforce stack_guard_gap */ prev = vma->vm_prev; diff --git a/mm/shmem.c b/mm/shmem.c index 0905215fb016..2c012eee133d 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -2848,7 +2848,7 @@ static int shmem_create(struct inode *dir, struct dentry *dentry, umode_t mode, static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry) { struct inode *inode = d_inode(old_dentry); - int ret; + int ret = 0; /* * No ordinary (disk based) filesystem counts links as inodes; diff --git a/net/dsa/dsa2.c b/net/dsa/dsa2.c index a1917025e155..410f19148106 100644 --- a/net/dsa/dsa2.c +++ b/net/dsa/dsa2.c @@ -612,8 +612,8 @@ static int dsa_switch_parse_ports_of(struct dsa_switch *ds, { struct device_node *ports, *port; struct dsa_port *dp; + int err = 0; u32 reg; - int err; ports = of_get_child_by_name(dn, "ports"); if (!ports) { @@ -624,19 +624,23 @@ static int dsa_switch_parse_ports_of(struct dsa_switch *ds, for_each_available_child_of_node(ports, port) { err = of_property_read_u32(port, "reg", ®); if (err) - return err; + goto out_put_node; - if (reg >= ds->num_ports) - return -EINVAL; + if (reg >= ds->num_ports) { + err = -EINVAL; + goto out_put_node; + } dp = &ds->ports[reg]; err = dsa_port_parse_of(dp, port); if (err) - return err; + goto out_put_node; } - return 0; +out_put_node: + of_node_put(ports); + return err; } static int dsa_switch_parse_member_of(struct dsa_switch *ds, diff --git a/net/dsa/port.c b/net/dsa/port.c index 2a2a878b5ce3..c2261697ee83 100644 --- a/net/dsa/port.c +++ b/net/dsa/port.c @@ -292,6 +292,7 @@ static struct phy_device *dsa_port_get_phy_device(struct dsa_port *dp) return ERR_PTR(-EPROBE_DEFER); } + of_node_put(phy_dn); return phydev; } diff --git a/net/ipv4/cipso_ipv4.c b/net/ipv4/cipso_ipv4.c index 777fa3b7fb13..f0165c5f376b 100644 --- a/net/ipv4/cipso_ipv4.c +++ b/net/ipv4/cipso_ipv4.c @@ -667,7 +667,8 @@ static int cipso_v4_map_lvl_valid(const struct cipso_v4_doi *doi_def, u8 level) case CIPSO_V4_MAP_PASS: return 0; case CIPSO_V4_MAP_TRANS: - if (doi_def->map.std->lvl.cipso[level] < CIPSO_V4_INV_LVL) + if ((level < doi_def->map.std->lvl.cipso_size) && + (doi_def->map.std->lvl.cipso[level] < CIPSO_V4_INV_LVL)) return 0; break; } @@ -1735,13 +1736,26 @@ validate_return: */ void cipso_v4_error(struct sk_buff *skb, int error, u32 gateway) { + unsigned char optbuf[sizeof(struct ip_options) + 40]; + struct ip_options *opt = (struct ip_options *)optbuf; + if (ip_hdr(skb)->protocol == IPPROTO_ICMP || error != -EACCES) return; + /* + * We might be called above the IP layer, + * so we can not use icmp_send and IPCB here. + */ + + memset(opt, 0, sizeof(struct ip_options)); + opt->optlen = ip_hdr(skb)->ihl*4 - sizeof(struct iphdr); + if (__ip_options_compile(dev_net(skb->dev), opt, skb, NULL)) + return; + if (gateway) - icmp_send(skb, ICMP_DEST_UNREACH, ICMP_NET_ANO, 0); + __icmp_send(skb, ICMP_DEST_UNREACH, ICMP_NET_ANO, 0, opt); else - icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_ANO, 0); + __icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_ANO, 0, opt); } /** diff --git a/net/ipv4/fib_frontend.c b/net/ipv4/fib_frontend.c index fe4f6a624238..ed14ec245584 100644 --- a/net/ipv4/fib_frontend.c +++ b/net/ipv4/fib_frontend.c @@ -710,6 +710,10 @@ static int rtm_to_fib_config(struct net *net, struct sk_buff *skb, case RTA_GATEWAY: cfg->fc_gw = nla_get_be32(attr); break; + case RTA_VIA: + NL_SET_ERR_MSG(extack, "IPv4 does not support RTA_VIA attribute"); + err = -EINVAL; + goto errout; case RTA_PRIORITY: cfg->fc_priority = nla_get_u32(attr); break; diff --git a/net/ipv4/icmp.c b/net/ipv4/icmp.c index 065997f414e6..3f24414150e2 100644 --- a/net/ipv4/icmp.c +++ b/net/ipv4/icmp.c @@ -570,7 +570,8 @@ relookup_failed: * MUST reply to only the first fragment. */ -void icmp_send(struct sk_buff *skb_in, int type, int code, __be32 info) +void __icmp_send(struct sk_buff *skb_in, int type, int code, __be32 info, + const struct ip_options *opt) { struct iphdr *iph; int room; @@ -691,7 +692,7 @@ void icmp_send(struct sk_buff *skb_in, int type, int code, __be32 info) iph->tos; mark = IP4_REPLY_MARK(net, skb_in->mark); - if (ip_options_echo(net, &icmp_param.replyopts.opt.opt, skb_in)) + if (__ip_options_echo(net, &icmp_param.replyopts.opt.opt, skb_in, opt)) goto out_unlock; @@ -742,7 +743,7 @@ out_bh_enable: local_bh_enable(); out:; } -EXPORT_SYMBOL(icmp_send); +EXPORT_SYMBOL(__icmp_send); static void icmp_socket_deliver(struct sk_buff *skb, u32 info) diff --git a/net/ipv4/ip_input.c b/net/ipv4/ip_input.c index 51d8efba6de2..1f4737b77067 100644 --- a/net/ipv4/ip_input.c +++ b/net/ipv4/ip_input.c @@ -307,11 +307,10 @@ drop: } static int ip_rcv_finish_core(struct net *net, struct sock *sk, - struct sk_buff *skb) + struct sk_buff *skb, struct net_device *dev) { const struct iphdr *iph = ip_hdr(skb); int (*edemux)(struct sk_buff *skb); - struct net_device *dev = skb->dev; struct rtable *rt; int err; @@ -400,6 +399,7 @@ drop_error: static int ip_rcv_finish(struct net *net, struct sock *sk, struct sk_buff *skb) { + struct net_device *dev = skb->dev; int ret; /* if ingress device is enslaved to an L3 master device pass the @@ -409,7 +409,7 @@ static int ip_rcv_finish(struct net *net, struct sock *sk, struct sk_buff *skb) if (!skb) return NET_RX_SUCCESS; - ret = ip_rcv_finish_core(net, sk, skb); + ret = ip_rcv_finish_core(net, sk, skb, dev); if (ret != NET_RX_DROP) ret = dst_input(skb); return ret; @@ -545,6 +545,7 @@ static void ip_list_rcv_finish(struct net *net, struct sock *sk, INIT_LIST_HEAD(&sublist); list_for_each_entry_safe(skb, next, head, list) { + struct net_device *dev = skb->dev; struct dst_entry *dst; skb_list_del_init(skb); @@ -554,7 +555,7 @@ static void ip_list_rcv_finish(struct net *net, struct sock *sk, skb = l3mdev_ip_rcv(skb); if (!skb) continue; - if (ip_rcv_finish_core(net, sk, skb) == NET_RX_DROP) + if (ip_rcv_finish_core(net, sk, skb, dev) == NET_RX_DROP) continue; dst = skb_dst(skb); diff --git a/net/ipv4/ip_options.c b/net/ipv4/ip_options.c index ed194d46c00e..32a35043c9f5 100644 --- a/net/ipv4/ip_options.c +++ b/net/ipv4/ip_options.c @@ -251,8 +251,9 @@ static void spec_dst_fill(__be32 *spec_dst, struct sk_buff *skb) * If opt == NULL, then skb->data should point to IP header. */ -int ip_options_compile(struct net *net, - struct ip_options *opt, struct sk_buff *skb) +int __ip_options_compile(struct net *net, + struct ip_options *opt, struct sk_buff *skb, + __be32 *info) { __be32 spec_dst = htonl(INADDR_ANY); unsigned char *pp_ptr = NULL; @@ -468,11 +469,22 @@ eol: return 0; error: - if (skb) { - icmp_send(skb, ICMP_PARAMETERPROB, 0, htonl((pp_ptr-iph)<<24)); - } + if (info) + *info = htonl((pp_ptr-iph)<<24); return -EINVAL; } + +int ip_options_compile(struct net *net, + struct ip_options *opt, struct sk_buff *skb) +{ + int ret; + __be32 info; + + ret = __ip_options_compile(net, opt, skb, &info); + if (ret != 0 && skb) + icmp_send(skb, ICMP_PARAMETERPROB, 0, info); + return ret; +} EXPORT_SYMBOL(ip_options_compile); /* diff --git a/net/ipv4/netlink.c b/net/ipv4/netlink.c index f86bb4f06609..d8e3a1fb8e82 100644 --- a/net/ipv4/netlink.c +++ b/net/ipv4/netlink.c @@ -3,9 +3,10 @@ #include <linux/types.h> #include <net/net_namespace.h> #include <net/netlink.h> +#include <linux/in6.h> #include <net/ip.h> -int rtm_getroute_parse_ip_proto(struct nlattr *attr, u8 *ip_proto, +int rtm_getroute_parse_ip_proto(struct nlattr *attr, u8 *ip_proto, u8 family, struct netlink_ext_ack *extack) { *ip_proto = nla_get_u8(attr); @@ -13,11 +14,19 @@ int rtm_getroute_parse_ip_proto(struct nlattr *attr, u8 *ip_proto, switch (*ip_proto) { case IPPROTO_TCP: case IPPROTO_UDP: + return 0; case IPPROTO_ICMP: + if (family != AF_INET) + break; + return 0; +#if IS_ENABLED(CONFIG_IPV6) + case IPPROTO_ICMPV6: + if (family != AF_INET6) + break; return 0; - default: - NL_SET_ERR_MSG(extack, "Unsupported ip proto"); - return -EOPNOTSUPP; +#endif } + NL_SET_ERR_MSG(extack, "Unsupported ip proto"); + return -EOPNOTSUPP; } EXPORT_SYMBOL_GPL(rtm_getroute_parse_ip_proto); diff --git a/net/ipv4/route.c b/net/ipv4/route.c index 5163b64f8fb3..7bb9128c8363 100644 --- a/net/ipv4/route.c +++ b/net/ipv4/route.c @@ -2803,7 +2803,7 @@ static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh, if (tb[RTA_IP_PROTO]) { err = rtm_getroute_parse_ip_proto(tb[RTA_IP_PROTO], - &ip_proto, extack); + &ip_proto, AF_INET, extack); if (err) return err; } diff --git a/net/ipv6/route.c b/net/ipv6/route.c index ce15dc4ccbfa..8dad1d690b78 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -4182,6 +4182,10 @@ static int rtm_to_fib6_config(struct sk_buff *skb, struct nlmsghdr *nlh, cfg->fc_gateway = nla_get_in6_addr(tb[RTA_GATEWAY]); cfg->fc_flags |= RTF_GATEWAY; } + if (tb[RTA_VIA]) { + NL_SET_ERR_MSG(extack, "IPv6 does not support RTA_VIA attribute"); + goto errout; + } if (tb[RTA_DST]) { int plen = (rtm->rtm_dst_len + 7) >> 3; @@ -4889,7 +4893,8 @@ static int inet6_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh, if (tb[RTA_IP_PROTO]) { err = rtm_getroute_parse_ip_proto(tb[RTA_IP_PROTO], - &fl6.flowi6_proto, extack); + &fl6.flowi6_proto, AF_INET6, + extack); if (err) goto errout; } diff --git a/net/ipv6/sit.c b/net/ipv6/sit.c index e8a1dabef803..09e440e8dfae 100644 --- a/net/ipv6/sit.c +++ b/net/ipv6/sit.c @@ -1873,6 +1873,7 @@ static int __net_init sit_init_net(struct net *net) err_reg_dev: ipip6_dev_free(sitn->fb_tunnel_dev); + free_netdev(sitn->fb_tunnel_dev); err_alloc_dev: return err; } diff --git a/net/mpls/af_mpls.c b/net/mpls/af_mpls.c index 7d55d4c04088..fa763e2e50ec 100644 --- a/net/mpls/af_mpls.c +++ b/net/mpls/af_mpls.c @@ -1838,6 +1838,9 @@ static int rtm_to_route_config(struct sk_buff *skb, goto errout; break; } + case RTA_GATEWAY: + NL_SET_ERR_MSG(extack, "MPLS does not support RTA_GATEWAY attribute"); + goto errout; case RTA_VIA: { if (nla_get_via(nla, &cfg->rc_via_alen, diff --git a/net/netlabel/netlabel_kapi.c b/net/netlabel/netlabel_kapi.c index ea7c67050792..ee3e5b6471a6 100644 --- a/net/netlabel/netlabel_kapi.c +++ b/net/netlabel/netlabel_kapi.c @@ -903,7 +903,8 @@ int netlbl_bitmap_walk(const unsigned char *bitmap, u32 bitmap_len, (state == 0 && (byte & bitmask) == 0)) return bit_spot; - bit_spot++; + if (++bit_spot >= bitmap_len) + return -1; bitmask >>= 1; if (bitmask == 0) { byte = bitmap[++byte_offset]; diff --git a/net/nfc/llcp_commands.c b/net/nfc/llcp_commands.c index 6a196e438b6c..d1fc019e932e 100644 --- a/net/nfc/llcp_commands.c +++ b/net/nfc/llcp_commands.c @@ -419,6 +419,10 @@ int nfc_llcp_send_connect(struct nfc_llcp_sock *sock) sock->service_name, sock->service_name_len, &service_name_tlv_length); + if (!service_name_tlv) { + err = -ENOMEM; + goto error_tlv; + } size += service_name_tlv_length; } @@ -429,9 +433,17 @@ int nfc_llcp_send_connect(struct nfc_llcp_sock *sock) miux_tlv = nfc_llcp_build_tlv(LLCP_TLV_MIUX, (u8 *)&miux, 0, &miux_tlv_length); + if (!miux_tlv) { + err = -ENOMEM; + goto error_tlv; + } size += miux_tlv_length; rw_tlv = nfc_llcp_build_tlv(LLCP_TLV_RW, &rw, 0, &rw_tlv_length); + if (!rw_tlv) { + err = -ENOMEM; + goto error_tlv; + } size += rw_tlv_length; pr_debug("SKB size %d SN length %zu\n", size, sock->service_name_len); @@ -484,9 +496,17 @@ int nfc_llcp_send_cc(struct nfc_llcp_sock *sock) miux_tlv = nfc_llcp_build_tlv(LLCP_TLV_MIUX, (u8 *)&miux, 0, &miux_tlv_length); + if (!miux_tlv) { + err = -ENOMEM; + goto error_tlv; + } size += miux_tlv_length; rw_tlv = nfc_llcp_build_tlv(LLCP_TLV_RW, &rw, 0, &rw_tlv_length); + if (!rw_tlv) { + err = -ENOMEM; + goto error_tlv; + } size += rw_tlv_length; skb = llcp_allocate_pdu(sock, LLCP_PDU_CC, size); diff --git a/net/nfc/llcp_core.c b/net/nfc/llcp_core.c index ef4026a23e80..4fa015208aab 100644 --- a/net/nfc/llcp_core.c +++ b/net/nfc/llcp_core.c @@ -532,10 +532,10 @@ static u8 nfc_llcp_reserve_sdp_ssap(struct nfc_llcp_local *local) static int nfc_llcp_build_gb(struct nfc_llcp_local *local) { - u8 *gb_cur, *version_tlv, version, version_length; - u8 *lto_tlv, lto_length; - u8 *wks_tlv, wks_length; - u8 *miux_tlv, miux_length; + u8 *gb_cur, version, version_length; + u8 lto_length, wks_length, miux_length; + u8 *version_tlv = NULL, *lto_tlv = NULL, + *wks_tlv = NULL, *miux_tlv = NULL; __be16 wks = cpu_to_be16(local->local_wks); u8 gb_len = 0; int ret = 0; @@ -543,17 +543,33 @@ static int nfc_llcp_build_gb(struct nfc_llcp_local *local) version = LLCP_VERSION_11; version_tlv = nfc_llcp_build_tlv(LLCP_TLV_VERSION, &version, 1, &version_length); + if (!version_tlv) { + ret = -ENOMEM; + goto out; + } gb_len += version_length; lto_tlv = nfc_llcp_build_tlv(LLCP_TLV_LTO, &local->lto, 1, <o_length); + if (!lto_tlv) { + ret = -ENOMEM; + goto out; + } gb_len += lto_length; pr_debug("Local wks 0x%lx\n", local->local_wks); wks_tlv = nfc_llcp_build_tlv(LLCP_TLV_WKS, (u8 *)&wks, 2, &wks_length); + if (!wks_tlv) { + ret = -ENOMEM; + goto out; + } gb_len += wks_length; miux_tlv = nfc_llcp_build_tlv(LLCP_TLV_MIUX, (u8 *)&local->miux, 0, &miux_length); + if (!miux_tlv) { + ret = -ENOMEM; + goto out; + } gb_len += miux_length; gb_len += ARRAY_SIZE(llcp_magic); diff --git a/net/sched/act_ipt.c b/net/sched/act_ipt.c index 8af6c11d2482..faa1addf89b3 100644 --- a/net/sched/act_ipt.c +++ b/net/sched/act_ipt.c @@ -199,8 +199,7 @@ err3: err2: kfree(tname); err1: - if (ret == ACT_P_CREATED) - tcf_idr_release(*a, bind); + tcf_idr_release(*a, bind); return err; } diff --git a/net/sched/act_skbedit.c b/net/sched/act_skbedit.c index 64dba3708fce..cfceed28c333 100644 --- a/net/sched/act_skbedit.c +++ b/net/sched/act_skbedit.c @@ -189,8 +189,7 @@ static int tcf_skbedit_init(struct net *net, struct nlattr *nla, params_new = kzalloc(sizeof(*params_new), GFP_KERNEL); if (unlikely(!params_new)) { - if (ret == ACT_P_CREATED) - tcf_idr_release(*a, bind); + tcf_idr_release(*a, bind); return -ENOMEM; } diff --git a/net/sched/act_tunnel_key.c b/net/sched/act_tunnel_key.c index 8b43fe0130f7..3f943de9a2c9 100644 --- a/net/sched/act_tunnel_key.c +++ b/net/sched/act_tunnel_key.c @@ -377,7 +377,8 @@ static int tunnel_key_init(struct net *net, struct nlattr *nla, return ret; release_tun_meta: - dst_release(&metadata->dst); + if (metadata) + dst_release(&metadata->dst); err_out: if (exists) diff --git a/net/sched/sch_netem.c b/net/sched/sch_netem.c index 75046ec72144..cc9d8133afcd 100644 --- a/net/sched/sch_netem.c +++ b/net/sched/sch_netem.c @@ -447,6 +447,7 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch, int nb = 0; int count = 1; int rc = NET_XMIT_SUCCESS; + int rc_drop = NET_XMIT_DROP; /* Do not fool qdisc_drop_all() */ skb->prev = NULL; @@ -486,6 +487,7 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch, q->duplicate = 0; rootq->enqueue(skb2, rootq, to_free); q->duplicate = dupsave; + rc_drop = NET_XMIT_SUCCESS; } /* @@ -498,7 +500,7 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch, if (skb_is_gso(skb)) { segs = netem_segment(skb, sch, to_free); if (!segs) - return NET_XMIT_DROP; + return rc_drop; } else { segs = skb; } @@ -521,8 +523,10 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch, 1<<(prandom_u32() % 8); } - if (unlikely(sch->q.qlen >= sch->limit)) - return qdisc_drop_all(skb, sch, to_free); + if (unlikely(sch->q.qlen >= sch->limit)) { + qdisc_drop_all(skb, sch, to_free); + return rc_drop; + } qdisc_qstats_backlog_inc(sch, skb); diff --git a/net/sctp/chunk.c b/net/sctp/chunk.c index 64bef313d436..5cb7c1ff97e9 100644 --- a/net/sctp/chunk.c +++ b/net/sctp/chunk.c @@ -192,7 +192,7 @@ struct sctp_datamsg *sctp_datamsg_from_user(struct sctp_association *asoc, if (unlikely(!max_data)) { max_data = sctp_min_frag_point(sctp_sk(asoc->base.sk), sctp_datachk_len(&asoc->stream)); - pr_warn_ratelimited("%s: asoc:%p frag_point is zero, forcing max_data to default minimum (%Zu)", + pr_warn_ratelimited("%s: asoc:%p frag_point is zero, forcing max_data to default minimum (%zu)", __func__, asoc, max_data); } diff --git a/net/socket.c b/net/socket.c index d80d87a395ea..320f51b22b19 100644 --- a/net/socket.c +++ b/net/socket.c @@ -577,6 +577,7 @@ static void __sock_release(struct socket *sock, struct inode *inode) if (inode) inode_lock(inode); sock->ops->release(sock); + sock->sk = NULL; if (inode) inode_unlock(inode); sock->ops = NULL; diff --git a/net/tipc/socket.c b/net/tipc/socket.c index 684f2125fc6b..70343ac448b1 100644 --- a/net/tipc/socket.c +++ b/net/tipc/socket.c @@ -379,11 +379,13 @@ static int tipc_sk_sock_err(struct socket *sock, long *timeout) #define tipc_wait_for_cond(sock_, timeo_, condition_) \ ({ \ + DEFINE_WAIT_FUNC(wait_, woken_wake_function); \ struct sock *sk_; \ int rc_; \ \ while ((rc_ = !(condition_))) { \ - DEFINE_WAIT_FUNC(wait_, woken_wake_function); \ + /* coupled with smp_wmb() in tipc_sk_proto_rcv() */ \ + smp_rmb(); \ sk_ = (sock_)->sk; \ rc_ = tipc_sk_sock_err((sock_), timeo_); \ if (rc_) \ @@ -1983,6 +1985,8 @@ static void tipc_sk_proto_rcv(struct sock *sk, return; case SOCK_WAKEUP: tipc_dest_del(&tsk->cong_links, msg_orignode(hdr), 0); + /* coupled with smp_rmb() in tipc_wait_for_cond() */ + smp_wmb(); tsk->cong_link_cnt--; wakeup = true; break; diff --git a/samples/vfio-mdev/mbochs.c b/samples/vfio-mdev/mbochs.c index ca7960adf5a3..b038aa9f5a70 100644 --- a/samples/vfio-mdev/mbochs.c +++ b/samples/vfio-mdev/mbochs.c @@ -1448,13 +1448,13 @@ static int __init mbochs_dev_init(void) { int ret = 0; - ret = alloc_chrdev_region(&mbochs_devt, 0, MINORMASK, MBOCHS_NAME); + ret = alloc_chrdev_region(&mbochs_devt, 0, MINORMASK + 1, MBOCHS_NAME); if (ret < 0) { pr_err("Error: failed to register mbochs_dev, err: %d\n", ret); return ret; } cdev_init(&mbochs_cdev, &vd_fops); - cdev_add(&mbochs_cdev, mbochs_devt, MINORMASK); + cdev_add(&mbochs_cdev, mbochs_devt, MINORMASK + 1); pr_info("%s: major %d\n", __func__, MAJOR(mbochs_devt)); mbochs_class = class_create(THIS_MODULE, MBOCHS_CLASS_NAME); @@ -1483,7 +1483,7 @@ failed2: class_destroy(mbochs_class); failed1: cdev_del(&mbochs_cdev); - unregister_chrdev_region(mbochs_devt, MINORMASK); + unregister_chrdev_region(mbochs_devt, MINORMASK + 1); return ret; } @@ -1494,7 +1494,7 @@ static void __exit mbochs_dev_exit(void) device_unregister(&mbochs_dev); cdev_del(&mbochs_cdev); - unregister_chrdev_region(mbochs_devt, MINORMASK); + unregister_chrdev_region(mbochs_devt, MINORMASK + 1); class_destroy(mbochs_class); mbochs_class = NULL; } diff --git a/samples/vfio-mdev/mdpy.c b/samples/vfio-mdev/mdpy.c index 96e7969c473a..cc86bf6566e4 100644 --- a/samples/vfio-mdev/mdpy.c +++ b/samples/vfio-mdev/mdpy.c @@ -752,13 +752,13 @@ static int __init mdpy_dev_init(void) { int ret = 0; - ret = alloc_chrdev_region(&mdpy_devt, 0, MINORMASK, MDPY_NAME); + ret = alloc_chrdev_region(&mdpy_devt, 0, MINORMASK + 1, MDPY_NAME); if (ret < 0) { pr_err("Error: failed to register mdpy_dev, err: %d\n", ret); return ret; } cdev_init(&mdpy_cdev, &vd_fops); - cdev_add(&mdpy_cdev, mdpy_devt, MINORMASK); + cdev_add(&mdpy_cdev, mdpy_devt, MINORMASK + 1); pr_info("%s: major %d\n", __func__, MAJOR(mdpy_devt)); mdpy_class = class_create(THIS_MODULE, MDPY_CLASS_NAME); @@ -787,7 +787,7 @@ failed2: class_destroy(mdpy_class); failed1: cdev_del(&mdpy_cdev); - unregister_chrdev_region(mdpy_devt, MINORMASK); + unregister_chrdev_region(mdpy_devt, MINORMASK + 1); return ret; } @@ -798,7 +798,7 @@ static void __exit mdpy_dev_exit(void) device_unregister(&mdpy_dev); cdev_del(&mdpy_cdev); - unregister_chrdev_region(mdpy_devt, MINORMASK); + unregister_chrdev_region(mdpy_devt, MINORMASK + 1); class_destroy(mdpy_class); mdpy_class = NULL; } diff --git a/samples/vfio-mdev/mtty.c b/samples/vfio-mdev/mtty.c index f6732aa16bb1..1c77c370c92f 100644 --- a/samples/vfio-mdev/mtty.c +++ b/samples/vfio-mdev/mtty.c @@ -156,15 +156,15 @@ static const struct file_operations vd_fops = { /* function prototypes */ -static int mtty_trigger_interrupt(uuid_le uuid); +static int mtty_trigger_interrupt(const guid_t *uuid); /* Helper functions */ -static struct mdev_state *find_mdev_state_by_uuid(uuid_le uuid) +static struct mdev_state *find_mdev_state_by_uuid(const guid_t *uuid) { struct mdev_state *mds; list_for_each_entry(mds, &mdev_devices_list, next) { - if (uuid_le_cmp(mdev_uuid(mds->mdev), uuid) == 0) + if (guid_equal(mdev_uuid(mds->mdev), uuid)) return mds; } @@ -1032,7 +1032,7 @@ static int mtty_set_irqs(struct mdev_device *mdev, uint32_t flags, return ret; } -static int mtty_trigger_interrupt(uuid_le uuid) +static int mtty_trigger_interrupt(const guid_t *uuid) { int ret = -1; struct mdev_state *mdev_state; @@ -1442,7 +1442,8 @@ static int __init mtty_dev_init(void) idr_init(&mtty_dev.vd_idr); - ret = alloc_chrdev_region(&mtty_dev.vd_devt, 0, MINORMASK, MTTY_NAME); + ret = alloc_chrdev_region(&mtty_dev.vd_devt, 0, MINORMASK + 1, + MTTY_NAME); if (ret < 0) { pr_err("Error: failed to register mtty_dev, err:%d\n", ret); @@ -1450,7 +1451,7 @@ static int __init mtty_dev_init(void) } cdev_init(&mtty_dev.vd_cdev, &vd_fops); - cdev_add(&mtty_dev.vd_cdev, mtty_dev.vd_devt, MINORMASK); + cdev_add(&mtty_dev.vd_cdev, mtty_dev.vd_devt, MINORMASK + 1); pr_info("major_number:%d\n", MAJOR(mtty_dev.vd_devt)); @@ -1487,7 +1488,7 @@ failed2: failed1: cdev_del(&mtty_dev.vd_cdev); - unregister_chrdev_region(mtty_dev.vd_devt, MINORMASK); + unregister_chrdev_region(mtty_dev.vd_devt, MINORMASK + 1); all_done: return ret; @@ -1501,7 +1502,7 @@ static void __exit mtty_dev_exit(void) device_unregister(&mtty_dev.dev); idr_destroy(&mtty_dev.vd_idr); cdev_del(&mtty_dev.vd_cdev); - unregister_chrdev_region(mtty_dev.vd_devt, MINORMASK); + unregister_chrdev_region(mtty_dev.vd_devt, MINORMASK + 1); class_destroy(mtty_dev.vd_class); mtty_dev.vd_class = NULL; pr_info("mtty_dev: Unloaded!\n"); diff --git a/scripts/Makefile.kasan b/scripts/Makefile.kasan index 25c259df8ffa..6deabedc67fc 100644 --- a/scripts/Makefile.kasan +++ b/scripts/Makefile.kasan @@ -26,7 +26,7 @@ else CFLAGS_KASAN := $(CFLAGS_KASAN_SHADOW) \ $(call cc-param,asan-globals=1) \ $(call cc-param,asan-instrumentation-with-call-threshold=$(call_threshold)) \ - $(call cc-param,asan-stack=1) \ + $(call cc-param,asan-stack=$(CONFIG_KASAN_STACK)) \ $(call cc-param,asan-use-after-scope=1) \ $(call cc-param,asan-instrument-allocas=1) endif diff --git a/security/integrity/iint.c b/security/integrity/iint.c index 88f04b3380d4..423876fca8b4 100644 --- a/security/integrity/iint.c +++ b/security/integrity/iint.c @@ -200,7 +200,7 @@ int integrity_kernel_read(struct file *file, loff_t offset, return -EBADF; old_fs = get_fs(); - set_fs(get_ds()); + set_fs(KERNEL_DS); ret = __vfs_read(file, buf, count, &offset); set_fs(old_fs); diff --git a/tools/testing/selftests/net/pmtu.sh b/tools/testing/selftests/net/pmtu.sh index e2c94e47707c..912b2dc50be3 100755 --- a/tools/testing/selftests/net/pmtu.sh +++ b/tools/testing/selftests/net/pmtu.sh @@ -103,6 +103,15 @@ # and check that configured MTU is used on link creation and changes, and # that MTU is properly calculated instead when MTU is not configured from # userspace +# +# - cleanup_ipv4_exception +# Similar to pmtu_ipv4_vxlan4_exception, but explicitly generate PMTU +# exceptions on multiple CPUs and check that the veth device tear-down +# happens in a timely manner +# +# - cleanup_ipv6_exception +# Same as above, but use IPv6 transport from A to B + # Kselftest framework requirement - SKIP code is 4. ksft_skip=4 @@ -135,7 +144,9 @@ tests=" pmtu_vti6_default_mtu vti6: default MTU assignment pmtu_vti4_link_add_mtu vti4: MTU setting on link creation pmtu_vti6_link_add_mtu vti6: MTU setting on link creation - pmtu_vti6_link_change_mtu vti6: MTU changes on link changes" + pmtu_vti6_link_change_mtu vti6: MTU changes on link changes + cleanup_ipv4_exception ipv4: cleanup of cached exceptions + cleanup_ipv6_exception ipv6: cleanup of cached exceptions" NS_A="ns-$(mktemp -u XXXXXX)" NS_B="ns-$(mktemp -u XXXXXX)" @@ -263,8 +274,6 @@ setup_fou_or_gue() { ${ns_a} ip link set ${encap}_a up ${ns_b} ip link set ${encap}_b up - - sleep 1 } setup_fou44() { @@ -302,6 +311,10 @@ setup_gue66() { setup_namespaces() { for n in ${NS_A} ${NS_B} ${NS_R1} ${NS_R2}; do ip netns add ${n} || return 1 + + # Disable DAD, so that we don't have to wait to use the + # configured IPv6 addresses + ip netns exec ${n} sysctl -q net/ipv6/conf/default/accept_dad=0 done } @@ -337,8 +350,6 @@ setup_vti() { ${ns_a} ip link set vti${proto}_a up ${ns_b} ip link set vti${proto}_b up - - sleep 1 } setup_vti4() { @@ -375,8 +386,6 @@ setup_vxlan_or_geneve() { ${ns_a} ip link set ${type}_a up ${ns_b} ip link set ${type}_b up - - sleep 1 } setup_geneve4() { @@ -588,8 +597,8 @@ test_pmtu_ipvX() { mtu "${ns_b}" veth_B-R2 1500 # Create route exceptions - ${ns_a} ${ping} -q -M want -i 0.1 -w 2 -s 1800 ${dst1} > /dev/null - ${ns_a} ${ping} -q -M want -i 0.1 -w 2 -s 1800 ${dst2} > /dev/null + ${ns_a} ${ping} -q -M want -i 0.1 -w 1 -s 1800 ${dst1} > /dev/null + ${ns_a} ${ping} -q -M want -i 0.1 -w 1 -s 1800 ${dst2} > /dev/null # Check that exceptions have been created with the correct PMTU pmtu_1="$(route_get_dst_pmtu_from_exception "${ns_a}" ${dst1})" @@ -621,7 +630,7 @@ test_pmtu_ipvX() { # Decrease remote MTU on path via R2, get new exception mtu "${ns_r2}" veth_R2-B 400 mtu "${ns_b}" veth_B-R2 400 - ${ns_a} ${ping} -q -M want -i 0.1 -w 2 -s 1400 ${dst2} > /dev/null + ${ns_a} ${ping} -q -M want -i 0.1 -w 1 -s 1400 ${dst2} > /dev/null pmtu_2="$(route_get_dst_pmtu_from_exception "${ns_a}" ${dst2})" check_pmtu_value "lock 552" "${pmtu_2}" "exceeding MTU, with MTU < min_pmtu" || return 1 @@ -638,7 +647,7 @@ test_pmtu_ipvX() { check_pmtu_value "1500" "${pmtu_2}" "increasing local MTU" || return 1 # Get new exception - ${ns_a} ${ping} -q -M want -i 0.1 -w 2 -s 1400 ${dst2} > /dev/null + ${ns_a} ${ping} -q -M want -i 0.1 -w 1 -s 1400 ${dst2} > /dev/null pmtu_2="$(route_get_dst_pmtu_from_exception "${ns_a}" ${dst2})" check_pmtu_value "lock 552" "${pmtu_2}" "exceeding MTU, with MTU < min_pmtu" || return 1 } @@ -687,7 +696,7 @@ test_pmtu_ipvX_over_vxlanY_or_geneveY_exception() { mtu "${ns_a}" ${type}_a $((${ll_mtu} + 1000)) mtu "${ns_b}" ${type}_b $((${ll_mtu} + 1000)) - ${ns_a} ${ping} -q -M want -i 0.1 -w 2 -s $((${ll_mtu} + 500)) ${dst} > /dev/null + ${ns_a} ${ping} -q -M want -i 0.1 -w 1 -s $((${ll_mtu} + 500)) ${dst} > /dev/null # Check that exception was created pmtu="$(route_get_dst_pmtu_from_exception "${ns_a}" ${dst})" @@ -767,7 +776,7 @@ test_pmtu_ipvX_over_fouY_or_gueY() { mtu "${ns_a}" ${encap}_a $((${ll_mtu} + 1000)) mtu "${ns_b}" ${encap}_b $((${ll_mtu} + 1000)) - ${ns_a} ${ping} -q -M want -i 0.1 -w 2 -s $((${ll_mtu} + 500)) ${dst} > /dev/null + ${ns_a} ${ping} -q -M want -i 0.1 -w 1 -s $((${ll_mtu} + 500)) ${dst} > /dev/null # Check that exception was created pmtu="$(route_get_dst_pmtu_from_exception "${ns_a}" ${dst})" @@ -825,13 +834,13 @@ test_pmtu_vti4_exception() { # Send DF packet without exceeding link layer MTU, check that no # exception is created - ${ns_a} ping -q -M want -i 0.1 -w 2 -s ${ping_payload} ${tunnel4_b_addr} > /dev/null + ${ns_a} ping -q -M want -i 0.1 -w 1 -s ${ping_payload} ${tunnel4_b_addr} > /dev/null pmtu="$(route_get_dst_pmtu_from_exception "${ns_a}" ${tunnel4_b_addr})" check_pmtu_value "" "${pmtu}" "sending packet smaller than PMTU (IP payload length ${esp_payload_rfc4106})" || return 1 # Now exceed link layer MTU by one byte, check that exception is created # with the right PMTU value - ${ns_a} ping -q -M want -i 0.1 -w 2 -s $((ping_payload + 1)) ${tunnel4_b_addr} > /dev/null + ${ns_a} ping -q -M want -i 0.1 -w 1 -s $((ping_payload + 1)) ${tunnel4_b_addr} > /dev/null pmtu="$(route_get_dst_pmtu_from_exception "${ns_a}" ${tunnel4_b_addr})" check_pmtu_value "${esp_payload_rfc4106}" "${pmtu}" "exceeding PMTU (IP payload length $((esp_payload_rfc4106 + 1)))" } @@ -847,7 +856,7 @@ test_pmtu_vti6_exception() { mtu "${ns_b}" veth_b 4000 mtu "${ns_a}" vti6_a 5000 mtu "${ns_b}" vti6_b 5000 - ${ns_a} ${ping6} -q -i 0.1 -w 2 -s 60000 ${tunnel6_b_addr} > /dev/null + ${ns_a} ${ping6} -q -i 0.1 -w 1 -s 60000 ${tunnel6_b_addr} > /dev/null # Check that exception was created pmtu="$(route_get_dst_pmtu_from_exception "${ns_a}" ${tunnel6_b_addr})" @@ -1008,6 +1017,61 @@ test_pmtu_vti6_link_change_mtu() { return ${fail} } +check_command() { + cmd=${1} + + if ! which ${cmd} > /dev/null 2>&1; then + err " missing required command: '${cmd}'" + return 1 + fi + return 0 +} + +test_cleanup_vxlanX_exception() { + outer="${1}" + encap="vxlan" + ll_mtu=4000 + + check_command taskset || return 2 + cpu_list=$(grep -m 2 processor /proc/cpuinfo | cut -d ' ' -f 2) + + setup namespaces routing ${encap}${outer} || return 2 + trace "${ns_a}" ${encap}_a "${ns_b}" ${encap}_b \ + "${ns_a}" veth_A-R1 "${ns_r1}" veth_R1-A \ + "${ns_b}" veth_B-R1 "${ns_r1}" veth_R1-B + + # Create route exception by exceeding link layer MTU + mtu "${ns_a}" veth_A-R1 $((${ll_mtu} + 1000)) + mtu "${ns_r1}" veth_R1-A $((${ll_mtu} + 1000)) + mtu "${ns_b}" veth_B-R1 ${ll_mtu} + mtu "${ns_r1}" veth_R1-B ${ll_mtu} + + mtu "${ns_a}" ${encap}_a $((${ll_mtu} + 1000)) + mtu "${ns_b}" ${encap}_b $((${ll_mtu} + 1000)) + + # Fill exception cache for multiple CPUs (2) + # we can always use inner IPv4 for that + for cpu in ${cpu_list}; do + taskset --cpu-list ${cpu} ${ns_a} ping -q -M want -i 0.1 -w 1 -s $((${ll_mtu} + 500)) ${tunnel4_b_addr} > /dev/null + done + + ${ns_a} ip link del dev veth_A-R1 & + iplink_pid=$! + sleep 1 + if [ "$(cat /proc/${iplink_pid}/cmdline 2>/dev/null | tr -d '\0')" = "iplinkdeldevveth_A-R1" ]; then + err " can't delete veth device in a timely manner, PMTU dst likely leaked" + return 1 + fi +} + +test_cleanup_ipv6_exception() { + test_cleanup_vxlanX_exception 6 +} + +test_cleanup_ipv4_exception() { + test_cleanup_vxlanX_exception 4 +} + usage() { echo echo "$0 [OPTIONS] [TEST]..." diff --git a/tools/testing/selftests/net/udpgro.sh b/tools/testing/selftests/net/udpgro.sh index aeac53a99aeb..ac2a30be9b32 100755 --- a/tools/testing/selftests/net/udpgro.sh +++ b/tools/testing/selftests/net/udpgro.sh @@ -37,7 +37,7 @@ run_one() { cfg_veth - ip netns exec "${PEER_NS}" ./udpgso_bench_rx ${rx_args} && \ + ip netns exec "${PEER_NS}" ./udpgso_bench_rx -C 1000 -R 10 ${rx_args} && \ echo "ok" || \ echo "failed" & @@ -81,7 +81,7 @@ run_one_nat() { # will land on the 'plain' one ip netns exec "${PEER_NS}" ./udpgso_bench_rx -G ${family} -b ${addr1} -n 0 & pid=$! - ip netns exec "${PEER_NS}" ./udpgso_bench_rx ${family} -b ${addr2%/*} ${rx_args} && \ + ip netns exec "${PEER_NS}" ./udpgso_bench_rx -C 1000 -R 10 ${family} -b ${addr2%/*} ${rx_args} && \ echo "ok" || \ echo "failed"& @@ -99,8 +99,8 @@ run_one_2sock() { cfg_veth - ip netns exec "${PEER_NS}" ./udpgso_bench_rx ${rx_args} -p 12345 & - ip netns exec "${PEER_NS}" ./udpgso_bench_rx ${rx_args} && \ + ip netns exec "${PEER_NS}" ./udpgso_bench_rx -C 1000 -R 10 ${rx_args} -p 12345 & + ip netns exec "${PEER_NS}" ./udpgso_bench_rx -C 2000 -R 10 ${rx_args} && \ echo "ok" || \ echo "failed" & diff --git a/tools/testing/selftests/net/udpgso_bench_rx.c b/tools/testing/selftests/net/udpgso_bench_rx.c index 0c960f673324..db3d4a8b5a4c 100644 --- a/tools/testing/selftests/net/udpgso_bench_rx.c +++ b/tools/testing/selftests/net/udpgso_bench_rx.c @@ -45,6 +45,8 @@ static int cfg_alen = sizeof(struct sockaddr_in6); static int cfg_expected_pkt_nr; static int cfg_expected_pkt_len; static int cfg_expected_gso_size; +static int cfg_connect_timeout_ms; +static int cfg_rcv_timeout_ms; static struct sockaddr_storage cfg_bind_addr; static bool interrupted; @@ -87,7 +89,7 @@ static unsigned long gettimeofday_ms(void) return (tv.tv_sec * 1000) + (tv.tv_usec / 1000); } -static void do_poll(int fd) +static void do_poll(int fd, int timeout_ms) { struct pollfd pfd; int ret; @@ -102,8 +104,16 @@ static void do_poll(int fd) break; if (ret == -1) error(1, errno, "poll"); - if (ret == 0) - continue; + if (ret == 0) { + if (!timeout_ms) + continue; + + timeout_ms -= 10; + if (timeout_ms <= 0) { + interrupted = true; + break; + } + } if (pfd.revents != POLLIN) error(1, errno, "poll: 0x%x expected 0x%x\n", pfd.revents, POLLIN); @@ -134,7 +144,7 @@ static int do_socket(bool do_tcp) if (listen(accept_fd, 1)) error(1, errno, "listen"); - do_poll(accept_fd); + do_poll(accept_fd, cfg_connect_timeout_ms); if (interrupted) exit(0); @@ -273,7 +283,9 @@ static void do_flush_udp(int fd) static void usage(const char *filepath) { - error(1, 0, "Usage: %s [-Grtv] [-b addr] [-p port] [-l pktlen] [-n packetnr] [-S gsosize]", filepath); + error(1, 0, "Usage: %s [-C connect_timeout] [-Grtv] [-b addr] [-p port]" + " [-l pktlen] [-n packetnr] [-R rcv_timeout] [-S gsosize]", + filepath); } static void parse_opts(int argc, char **argv) @@ -282,7 +294,7 @@ static void parse_opts(int argc, char **argv) /* bind to any by default */ setup_sockaddr(PF_INET6, "::", &cfg_bind_addr); - while ((c = getopt(argc, argv, "4b:Gl:n:p:rS:tv")) != -1) { + while ((c = getopt(argc, argv, "4b:C:Gl:n:p:rR:S:tv")) != -1) { switch (c) { case '4': cfg_family = PF_INET; @@ -292,6 +304,9 @@ static void parse_opts(int argc, char **argv) case 'b': setup_sockaddr(cfg_family, optarg, &cfg_bind_addr); break; + case 'C': + cfg_connect_timeout_ms = strtoul(optarg, NULL, 0); + break; case 'G': cfg_gro_segment = true; break; @@ -307,6 +322,9 @@ static void parse_opts(int argc, char **argv) case 'r': cfg_read_all = true; break; + case 'R': + cfg_rcv_timeout_ms = strtoul(optarg, NULL, 0); + break; case 'S': cfg_expected_gso_size = strtol(optarg, NULL, 0); break; @@ -329,8 +347,9 @@ static void parse_opts(int argc, char **argv) static void do_recv(void) { + int timeout_ms = cfg_tcp ? cfg_rcv_timeout_ms : cfg_connect_timeout_ms; unsigned long tnow, treport; - int fd, loop = 0; + int fd; fd = do_socket(cfg_tcp); @@ -342,12 +361,7 @@ static void do_recv(void) treport = gettimeofday_ms() + 1000; do { - /* force termination after the second poll(); this cope both - * with sender slower than receiver and missing packet errors - */ - if (cfg_expected_pkt_nr && loop++) - interrupted = true; - do_poll(fd); + do_poll(fd, timeout_ms); if (cfg_tcp) do_flush_tcp(fd); @@ -365,6 +379,8 @@ static void do_recv(void) treport = tnow + 1000; } + timeout_ms = cfg_rcv_timeout_ms; + } while (!interrupted); if (cfg_expected_pkt_nr && (packets != cfg_expected_pkt_nr)) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 585845203db8..076bc38963bf 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -4044,7 +4044,7 @@ static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm) } add_uevent_var(env, "PID=%d", kvm->userspace_pid); - if (kvm->debugfs_dentry) { + if (!IS_ERR_OR_NULL(kvm->debugfs_dentry)) { char *tmp, *p = kmalloc(PATH_MAX, GFP_KERNEL); if (p) { |