[PATCH v13 03/26] KVM: x86: Replace nr_vcpus_matched_tsc count with all_vcpus_matched_tsc bool

[David Woodhouse <dwmw2@xxxxxxxxxxxxx>]

From: David Woodhouse <dwmw@xxxxxxxxxxxx>

Comparing nr_vcpus_matched_tsc against kvm->online_vcpus at read time
was always racy — a new vCPU can appear between the increment and its
first synchronization — and it made "all vCPUs matched" an accident of
arithmetic rather than a property with defined semantics. Replace it
with a boolean maintained at synchronization time under tsc_write_lock:
set in kvm_track_tsc_matching(), cleared when a new generation opens,
consumed by pvclock_update_vm_gtod_copy(). The count remains as the
input from which the flag is computed, and for the trace event.

The consistent interface is the point. The flag starts true at VM
creation, so a VM whose vCPUs never synchronize — a protected guest
TSC (TDX, SNP Secure TSC) returns early from __kvm_synchronize_tsc() —
is eligible for PVCLOCK_TSC_STABLE_BIT by design rather than by luck
of the read-time arithmetic, and use_master_clock is true from VM
creation on a TSC-clocksource host, before any vCPU exists; later
patches build on exactly that. The default scaling ratio must then be
initialized too (a zero ratio zeroes tsc_hz and turns the master clock
straight back off), and the khz/ratio bookkeeping recorded even for a
protected guest TSC: it is host-side state, which a scaled protected
guest must not leave at the host's unscaled frequency.

The cached state needs care at vCPU creation: the new vCPU is counted
in online_vcpus before it is synchronized, and userspace taking
vcpu->mutex first can hold that window open across a whole first
KVM_RUN. Clear the matched state up front in postcreate — the old
read-time comparison failed conservatively false throughout — and let
the synchronization (or a racing host-initiated TSC write on the
already-live fd) re-latch it; a postcreate aborted by a fatal signal
just propagates whatever the state then is with a masterclock update.
Only the instants between the online_vcpus increment and the clear
remain stale-true.

The explicit clear at new-generation start is currently overwritten by
the unconditional recompute in kvm_track_tsc_matching(); it is
preparation for later patches which consume the previous value.

Signed-off-by: David Woodhouse <dwmw@xxxxxxxxxxxx>
---
 arch/x86/include/asm/kvm_host.h |  6 ++-
 arch/x86/kvm/x86.c              | 76 +++++++++++++++++++++++++++------
 2 files changed, 67 insertions(+), 15 deletions(-)

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index ea48b1949316..18d7bb9fd8d2 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -1232,8 +1232,9 @@ struct kvm_arch {
        s64 kvmclock_offset;
 
        /*
-        * This also protects nr_vcpus_matched_tsc which is read from a
-        * preemption-disabled region, so it must be a raw spinlock.
+        * This also protects nr_vcpus_matched_tsc and
+        * all_vcpus_matched_tsc, which are read from preemption-disabled
+        * regions, so it must be a raw spinlock.
         */
        raw_spinlock_t tsc_write_lock;
        u32 cur_tsc_khz;
@@ -1242,6 +1243,7 @@ struct kvm_arch {
        u64 cur_tsc_write;
        u64 cur_tsc_offset;
        u64 cur_tsc_generation;
+       bool all_vcpus_matched_tsc;
        int nr_vcpus_matched_tsc;
 
        u32 default_tsc_khz;
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 12f465040efc..1bb0136042a6 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -1125,8 +1125,10 @@ static void kvm_track_tsc_matching(struct kvm_vcpu 
*vcpu, bool new_generation)
         * and all vCPUs must have matching TSCs.  Note, the count for matching
         * vCPUs doesn't include the reference vCPU, hence "+1".
         */
-       bool use_master_clock = (ka->nr_vcpus_matched_tsc + 1 ==
-                                atomic_read(&vcpu->kvm->online_vcpus)) &&
+       ka->all_vcpus_matched_tsc = (ka->nr_vcpus_matched_tsc + 1 ==
+                                    atomic_read(&vcpu->kvm->online_vcpus));
+
+       bool use_master_clock = ka->all_vcpus_matched_tsc &&
                                gtod_is_based_on_tsc(gtod->clock.vclock_mode);
 
        /*
@@ -1327,20 +1329,23 @@ static void __kvm_synchronize_tsc(struct kvm_vcpu 
*vcpu, u64 offset, u64 tsc,
 
        lockdep_assert_held(&kvm->arch.tsc_write_lock);
 
-       if (vcpu->arch.guest_tsc_protected)
-               return;
-
-       if (user_set_tsc)
-               vcpu->kvm->arch.user_set_tsc = true;
-
        /*
-        * Track the TSC frequency, scaling ratio, and offset for the current
+        * Track the TSC frequency and scaling ratio for the current
         * generation. These are used to detect matching TSC writes and to
-        * compute the guest TSC from the host clock.
+        * compute the guest TSC from the host clock. This is host-side
+        * bookkeeping, recorded even for a protected guest TSC — a scaled
+        * protected guest (TDX, SNP Secure TSC) must not leave the master
+        * clock computing with the host's unscaled frequency.
         */
        kvm->arch.cur_tsc_khz = vcpu->arch.virtual_tsc_khz;
        kvm->arch.cur_tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio;
 
+       if (vcpu->arch.guest_tsc_protected)
+               return;
+
+       if (user_set_tsc)
+               vcpu->kvm->arch.user_set_tsc = true;
+
        vcpu->arch.last_guest_tsc = tsc;
 
        /*
@@ -1367,6 +1372,7 @@ static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, 
u64 offset, u64 tsc,
                kvm->arch.cur_tsc_write = tsc;
                kvm->arch.cur_tsc_offset = offset;
                kvm->arch.nr_vcpus_matched_tsc = 0;
+               kvm->arch.all_vcpus_matched_tsc = false;
        } else if (vcpu->arch.this_tsc_generation != 
kvm->arch.cur_tsc_generation) {
                kvm->arch.nr_vcpus_matched_tsc++;
        }
@@ -1593,8 +1599,7 @@ static void pvclock_update_vm_gtod_copy(struct kvm *kvm)
        bool host_tsc_clocksource, vcpus_matched;
 
        lockdep_assert_held(&kvm->arch.tsc_write_lock);
-       vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
-                       atomic_read(&kvm->online_vcpus));
+       vcpus_matched = ka->all_vcpus_matched_tsc;
 
        /*
         * If the host uses TSC clock, then passthrough TSC as stable
@@ -9584,8 +9589,38 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
 
 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
 {
-       if (mutex_lock_killable(&vcpu->mutex))
+       struct kvm_arch *ka = &vcpu->kvm->arch;
+       unsigned long flags;
+
+       /*
+        * The new vCPU is already counted in online_vcpus but has not
+        * been synchronized. Clear the cached matched state before
+        * trying the mutex: the old read-time comparison failed
+        * conservatively false throughout this window, and userspace
+        * can hold it open across a whole first KVM_RUN by taking
+        * vcpu->mutex before we do. The synchronization below (or any
+        * racing host-initiated TSC write on the already-live fd)
+        * re-latches the flags. A protected guest TSC never
+        * participates in offset matching, so for it there is nothing
+        * to clear — and no later sync to re-latch.
+        */
+       if (!vcpu->arch.guest_tsc_protected) {
+               raw_spin_lock_irqsave(&ka->tsc_write_lock, flags);
+               WRITE_ONCE(ka->all_vcpus_matched_tsc, false);
+               raw_spin_unlock_irqrestore(&ka->tsc_write_lock, flags);
+       }
+
+       if (mutex_lock_killable(&vcpu->mutex)) {
+               /*
+                * Aborted by a fatal signal: the synchronization never
+                * ran. Propagate whatever the matched state now is (a
+                * racing TSC write may have re-latched it), so the
+                * master clock does not stay in its previous mode with
+                * running siblings keeping PVCLOCK_TSC_STABLE_BIT.
+                */
+               kvm_update_masterclock(vcpu->kvm);
                return;
+       }
        vcpu_load(vcpu);
        kvm_synchronize_tsc(vcpu, NULL);
        if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))
@@ -10061,6 +10096,21 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long 
type)
        ratelimit_set_flags(&kvm->arch.kvmclock_update_rs, 
RATELIMIT_MSG_ON_RELEASE);
        kvm->arch.kvmclock_offset = -get_kvmclock_base_ns();
 
+       /*
+        * Start with all (zero) vCPUs considered matched, so that a VM whose
+        * vCPUs never reach kvm_track_tsc_matching() — notably when the guest
+        * TSC is protected (TDX, SNP Secure TSC) and __kvm_synchronize_tsc()
+        * returns early — is still eligible for PVCLOCK_TSC_STABLE_BIT. The
+        * scaling ratio must then also be initialized: with vcpus_matched
+        * now true here, pvclock_update_vm_gtod_copy() reaches the scaling
+        * path, and a zero ratio would zero tsc_hz and turn the master
+        * clock straight back off (permanently so for protected-TSC guests,
+        * whose syncs return before kvm_track_tsc_matching() ever requests
+        * the masterclock update that would recompute it).
+        */
+       kvm->arch.all_vcpus_matched_tsc = true;
+       kvm->arch.cur_tsc_scaling_ratio = kvm_caps.default_tsc_scaling_ratio;
+
        raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
        pvclock_update_vm_gtod_copy(kvm);
        raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
-- 
2.43.0