diff options
Diffstat (limited to 'arch/x86/coco/tdx')
| -rw-r--r-- | arch/x86/coco/tdx/tdcall.S | 96 | ||||
| -rw-r--r-- | arch/x86/coco/tdx/tdx.c | 87 |
2 files changed, 127 insertions, 56 deletions
diff --git a/arch/x86/coco/tdx/tdcall.S b/arch/x86/coco/tdx/tdcall.S index f9eb1134f22d..6a255e6809bc 100644 --- a/arch/x86/coco/tdx/tdcall.S +++ b/arch/x86/coco/tdx/tdcall.S @@ -13,6 +13,12 @@ /* * Bitmasks of exposed registers (with VMM). */ +#define TDX_RDX BIT(2) +#define TDX_RBX BIT(3) +#define TDX_RSI BIT(6) +#define TDX_RDI BIT(7) +#define TDX_R8 BIT(8) +#define TDX_R9 BIT(9) #define TDX_R10 BIT(10) #define TDX_R11 BIT(11) #define TDX_R12 BIT(12) @@ -27,9 +33,11 @@ * details can be found in TDX GHCI specification, section * titled "TDCALL [TDG.VP.VMCALL] leaf". */ -#define TDVMCALL_EXPOSE_REGS_MASK ( TDX_R10 | TDX_R11 | \ - TDX_R12 | TDX_R13 | \ - TDX_R14 | TDX_R15 ) +#define TDVMCALL_EXPOSE_REGS_MASK \ + ( TDX_RDX | TDX_RBX | TDX_RSI | TDX_RDI | TDX_R8 | TDX_R9 | \ + TDX_R10 | TDX_R11 | TDX_R12 | TDX_R13 | TDX_R14 | TDX_R15 ) + +.section .noinstr.text, "ax" /* * __tdx_module_call() - Used by TDX guests to request services from @@ -124,38 +132,38 @@ SYM_FUNC_START(__tdx_hypercall) push %r14 push %r13 push %r12 + push %rbx + + /* Free RDI and RSI to be used as TDVMCALL arguments */ + movq %rdi, %rax + push %rsi + + /* Copy hypercall registers from arg struct: */ + movq TDX_HYPERCALL_r8(%rax), %r8 + movq TDX_HYPERCALL_r9(%rax), %r9 + movq TDX_HYPERCALL_r10(%rax), %r10 + movq TDX_HYPERCALL_r11(%rax), %r11 + movq TDX_HYPERCALL_r12(%rax), %r12 + movq TDX_HYPERCALL_r13(%rax), %r13 + movq TDX_HYPERCALL_r14(%rax), %r14 + movq TDX_HYPERCALL_r15(%rax), %r15 + movq TDX_HYPERCALL_rdi(%rax), %rdi + movq TDX_HYPERCALL_rsi(%rax), %rsi + movq TDX_HYPERCALL_rbx(%rax), %rbx + movq TDX_HYPERCALL_rdx(%rax), %rdx + + push %rax /* Mangle function call ABI into TDCALL ABI: */ /* Set TDCALL leaf ID (TDVMCALL (0)) in RAX */ xor %eax, %eax - /* Copy hypercall registers from arg struct: */ - movq TDX_HYPERCALL_r10(%rdi), %r10 - movq TDX_HYPERCALL_r11(%rdi), %r11 - movq TDX_HYPERCALL_r12(%rdi), %r12 - movq TDX_HYPERCALL_r13(%rdi), %r13 - movq TDX_HYPERCALL_r14(%rdi), %r14 - movq TDX_HYPERCALL_r15(%rdi), %r15 - movl $TDVMCALL_EXPOSE_REGS_MASK, %ecx - /* - * For the idle loop STI needs to be called directly before the TDCALL - * that enters idle (EXIT_REASON_HLT case). STI instruction enables - * interrupts only one instruction later. If there is a window between - * STI and the instruction that emulates the HALT state, there is a - * chance for interrupts to happen in this window, which can delay the - * HLT operation indefinitely. Since this is the not the desired - * result, conditionally call STI before TDCALL. - */ - testq $TDX_HCALL_ISSUE_STI, %rsi - jz .Lskip_sti - sti -.Lskip_sti: tdcall /* - * RAX==0 indicates a failure of the TDVMCALL mechanism itself and that + * RAX!=0 indicates a failure of the TDVMCALL mechanism itself and that * something has gone horribly wrong with the TDX module. * * The return status of the hypercall operation is in a separate @@ -165,30 +173,46 @@ SYM_FUNC_START(__tdx_hypercall) testq %rax, %rax jne .Lpanic - /* TDVMCALL leaf return code is in R10 */ - movq %r10, %rax + pop %rax /* Copy hypercall result registers to arg struct if needed */ - testq $TDX_HCALL_HAS_OUTPUT, %rsi + testq $TDX_HCALL_HAS_OUTPUT, (%rsp) jz .Lout - movq %r10, TDX_HYPERCALL_r10(%rdi) - movq %r11, TDX_HYPERCALL_r11(%rdi) - movq %r12, TDX_HYPERCALL_r12(%rdi) - movq %r13, TDX_HYPERCALL_r13(%rdi) - movq %r14, TDX_HYPERCALL_r14(%rdi) - movq %r15, TDX_HYPERCALL_r15(%rdi) + movq %r8, TDX_HYPERCALL_r8(%rax) + movq %r9, TDX_HYPERCALL_r9(%rax) + movq %r10, TDX_HYPERCALL_r10(%rax) + movq %r11, TDX_HYPERCALL_r11(%rax) + movq %r12, TDX_HYPERCALL_r12(%rax) + movq %r13, TDX_HYPERCALL_r13(%rax) + movq %r14, TDX_HYPERCALL_r14(%rax) + movq %r15, TDX_HYPERCALL_r15(%rax) + movq %rdi, TDX_HYPERCALL_rdi(%rax) + movq %rsi, TDX_HYPERCALL_rsi(%rax) + movq %rbx, TDX_HYPERCALL_rbx(%rax) + movq %rdx, TDX_HYPERCALL_rdx(%rax) .Lout: + /* TDVMCALL leaf return code is in R10 */ + movq %r10, %rax + /* * Zero out registers exposed to the VMM to avoid speculative execution * with VMM-controlled values. This needs to include all registers - * present in TDVMCALL_EXPOSE_REGS_MASK (except R12-R15). R12-R15 - * context will be restored. + * present in TDVMCALL_EXPOSE_REGS_MASK, except RBX, and R12-R15 which + * will be restored. */ + xor %r8d, %r8d + xor %r9d, %r9d xor %r10d, %r10d xor %r11d, %r11d + xor %rdi, %rdi + xor %rdx, %rdx + + /* Remove TDX_HCALL_* flags from the stack */ + pop %rsi /* Restore callee-saved GPRs as mandated by the x86_64 ABI */ + pop %rbx pop %r12 pop %r13 pop %r14 diff --git a/arch/x86/coco/tdx/tdx.c b/arch/x86/coco/tdx/tdx.c index 669d9e4f2901..055300e08fb3 100644 --- a/arch/x86/coco/tdx/tdx.c +++ b/arch/x86/coco/tdx/tdx.c @@ -19,9 +19,14 @@ #define TDX_GET_VEINFO 3 #define TDX_GET_REPORT 4 #define TDX_ACCEPT_PAGE 6 +#define TDX_WR 8 + +/* TDCS fields. To be used by TDG.VM.WR and TDG.VM.RD module calls */ +#define TDCS_NOTIFY_ENABLES 0x9100000000000010 /* TDX hypercall Leaf IDs */ #define TDVMCALL_MAP_GPA 0x10001 +#define TDVMCALL_REPORT_FATAL_ERROR 0x10003 /* MMIO direction */ #define EPT_READ 0 @@ -37,6 +42,7 @@ #define VE_GET_PORT_NUM(e) ((e) >> 16) #define VE_IS_IO_STRING(e) ((e) & BIT(4)) +#define ATTR_DEBUG BIT(0) #define ATTR_SEPT_VE_DISABLE BIT(28) /* TDX Module call error codes */ @@ -64,8 +70,9 @@ static inline u64 _tdx_hypercall(u64 fn, u64 r12, u64 r13, u64 r14, u64 r15) } /* Called from __tdx_hypercall() for unrecoverable failure */ -void __tdx_hypercall_failed(void) +noinstr void __tdx_hypercall_failed(void) { + instrumentation_begin(); panic("TDVMCALL failed. TDX module bug?"); } @@ -75,7 +82,7 @@ void __tdx_hypercall_failed(void) * Reusing the KVM EXIT_REASON macros makes it easier to connect the host and * guest sides of these calls. */ -static u64 hcall_func(u64 exit_reason) +static __always_inline u64 hcall_func(u64 exit_reason) { return exit_reason; } @@ -140,6 +147,41 @@ int tdx_mcall_get_report0(u8 *reportdata, u8 *tdreport) } EXPORT_SYMBOL_GPL(tdx_mcall_get_report0); +static void __noreturn tdx_panic(const char *msg) +{ + struct tdx_hypercall_args args = { + .r10 = TDX_HYPERCALL_STANDARD, + .r11 = TDVMCALL_REPORT_FATAL_ERROR, + .r12 = 0, /* Error code: 0 is Panic */ + }; + union { + /* Define register order according to the GHCI */ + struct { u64 r14, r15, rbx, rdi, rsi, r8, r9, rdx; }; + + char str[64]; + } message; + + /* VMM assumes '\0' in byte 65, if the message took all 64 bytes */ + strncpy(message.str, msg, 64); + + args.r8 = message.r8; + args.r9 = message.r9; + args.r14 = message.r14; + args.r15 = message.r15; + args.rdi = message.rdi; + args.rsi = message.rsi; + args.rbx = message.rbx; + args.rdx = message.rdx; + + /* + * This hypercall should never return and it is not safe + * to keep the guest running. Call it forever if it + * happens to return. + */ + while (1) + __tdx_hypercall(&args, 0); +} + static void tdx_parse_tdinfo(u64 *cc_mask) { struct tdx_module_output out; @@ -171,8 +213,15 @@ static void tdx_parse_tdinfo(u64 *cc_mask) * TD-private memory. Only VMM-shared memory (MMIO) will #VE. */ td_attr = out.rdx; - if (!(td_attr & ATTR_SEPT_VE_DISABLE)) - panic("TD misconfiguration: SEPT_VE_DISABLE attibute must be set.\n"); + if (!(td_attr & ATTR_SEPT_VE_DISABLE)) { + const char *msg = "TD misconfiguration: SEPT_VE_DISABLE attribute must be set."; + + /* Relax SEPT_VE_DISABLE check for debug TD. */ + if (td_attr & ATTR_DEBUG) + pr_warn("%s\n", msg); + else + tdx_panic(msg); + } } /* @@ -220,7 +269,7 @@ static int ve_instr_len(struct ve_info *ve) } } -static u64 __cpuidle __halt(const bool irq_disabled, const bool do_sti) +static u64 __cpuidle __halt(const bool irq_disabled) { struct tdx_hypercall_args args = { .r10 = TDX_HYPERCALL_STANDARD, @@ -240,20 +289,14 @@ static u64 __cpuidle __halt(const bool irq_disabled, const bool do_sti) * can keep the vCPU in virtual HLT, even if an IRQ is * pending, without hanging/breaking the guest. */ - return __tdx_hypercall(&args, do_sti ? TDX_HCALL_ISSUE_STI : 0); + return __tdx_hypercall(&args, 0); } static int handle_halt(struct ve_info *ve) { - /* - * Since non safe halt is mainly used in CPU offlining - * and the guest will always stay in the halt state, don't - * call the STI instruction (set do_sti as false). - */ const bool irq_disabled = irqs_disabled(); - const bool do_sti = false; - if (__halt(irq_disabled, do_sti)) + if (__halt(irq_disabled)) return -EIO; return ve_instr_len(ve); @@ -261,18 +304,12 @@ static int handle_halt(struct ve_info *ve) void __cpuidle tdx_safe_halt(void) { - /* - * For do_sti=true case, __tdx_hypercall() function enables - * interrupts using the STI instruction before the TDCALL. So - * set irq_disabled as false. - */ const bool irq_disabled = false; - const bool do_sti = true; /* * Use WARN_ONCE() to report the failure. */ - if (__halt(irq_disabled, do_sti)) + if (__halt(irq_disabled)) WARN_ONCE(1, "HLT instruction emulation failed\n"); } @@ -628,6 +665,11 @@ static int virt_exception_user(struct pt_regs *regs, struct ve_info *ve) } } +static inline bool is_private_gpa(u64 gpa) +{ + return gpa == cc_mkenc(gpa); +} + /* * Handle the kernel #VE. * @@ -646,6 +688,8 @@ static int virt_exception_kernel(struct pt_regs *regs, struct ve_info *ve) case EXIT_REASON_CPUID: return handle_cpuid(regs, ve); case EXIT_REASON_EPT_VIOLATION: + if (is_private_gpa(ve->gpa)) + panic("Unexpected EPT-violation on private memory."); return handle_mmio(regs, ve); case EXIT_REASON_IO_INSTRUCTION: return handle_io(regs, ve); @@ -812,6 +856,9 @@ void __init tdx_early_init(void) tdx_parse_tdinfo(&cc_mask); cc_set_mask(cc_mask); + /* Kernel does not use NOTIFY_ENABLES and does not need random #VEs */ + tdx_module_call(TDX_WR, 0, TDCS_NOTIFY_ENABLES, 0, -1ULL, NULL); + /* * All bits above GPA width are reserved and kernel treats shared bit * as flag, not as part of physical address. |