arch/arm/common/mcpm_head.S

/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * arch/arm/common/mcpm_head.S -- kernel entry point for multi-cluster PM
 *
 * Created by:  Nicolas Pitre, March 2012
 * Copyright:   (C) 2012-2013  Linaro Limited
 *
 * Refer to Documentation/arm/cluster-pm-race-avoidance.rst
 * for details of the synchronisation algorithms used here.
 */

#include <linux/linkage.h>
#include <asm/mcpm.h>
#include <asm/assembler.h>

#include "vlock.h"

.if MCPM_SYNC_CLUSTER_CPUS
.error "cpus must be the first member of struct mcpm_sync_struct"
.endif

	.macro	pr_dbg	string
#if defined(CONFIG_DEBUG_LL) && defined(DEBUG)
	b	1901f
1902:	.asciz	"CPU"
1903:	.asciz	" cluster"
1904:	.asciz	": \string"
	.align
1901:	adr	r0, 1902b
	bl	printascii
	mov	r0, r9
	bl	printhex2
	adr	r0, 1903b
	bl	printascii
	mov	r0, r10
	bl	printhex2
	adr	r0, 1904b
	bl	printascii
#endif
	.endm

	.arm
	.align

ENTRY(mcpm_entry_point)

 ARM_BE8(setend        be)
 THUMB(	badr	r12, 1f		)
 THUMB(	bx	r12		)
 THUMB(	.thumb			)
1:
	mrc	p15, 0, r0, c0, c0, 5		@ MPIDR
	ubfx	r9, r0, #0, #8			@ r9 = cpu
	ubfx	r10, r0, #8, #8			@ r10 = cluster
	mov	r3, #MAX_CPUS_PER_CLUSTER
	mla	r4, r3, r10, r9			@ r4 = canonical CPU index
	cmp	r4, #(MAX_CPUS_PER_CLUSTER * MAX_NR_CLUSTERS)
	blo	2f

	/* We didn't expect this CPU.  Try to cheaply make it quiet. */
1:	wfi
	wfe
	b	1b

2:	pr_dbg	"kernel mcpm_entry_point\n"

	/*
	 * MMU is off so we need to get to various variables in a
	 * position independent way.
	 */
	adr	r5, 3f
	ldmia	r5, {r0, r6, r7, r8, r11}
	add	r0, r5, r0			@ r0 = mcpm_entry_early_pokes
	add	r6, r5, r6			@ r6 = mcpm_entry_vectors
	ldr	r7, [r5, r7]			@ r7 = mcpm_power_up_setup_phys
	add	r8, r5, r8			@ r8 = mcpm_sync
	add	r11, r5, r11			@ r11 = first_man_locks

	@ Perform an early poke, if any
	add	r0, r0, r4, lsl #3
	ldmia	r0, {r0, r1}
	teq	r0, #0
	strne	r1, [r0]

	mov	r0, #MCPM_SYNC_CLUSTER_SIZE
	mla	r8, r0, r10, r8			@ r8 = sync cluster base

	@ Signal that this CPU is coming UP:
	mov	r0, #CPU_COMING_UP
	mov	r5, #MCPM_SYNC_CPU_SIZE
	mla	r5, r9, r5, r8			@ r5 = sync cpu address
	strb	r0, [r5]

	@ At this point, the cluster cannot unexpectedly enter the GOING_DOWN
	@ state, because there is at least one active CPU (this CPU).

	mov	r0, #VLOCK_SIZE
	mla	r11, r0, r10, r11		@ r11 = cluster first man lock
	mov	r0, r11
	mov	r1, r9				@ cpu
	bl	vlock_trylock			@ implies DMB

	cmp	r0, #0				@ failed to get the lock?
	bne	mcpm_setup_wait		@ wait for cluster setup if so

	ldrb	r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
	cmp	r0, #CLUSTER_UP			@ cluster already up?
	bne	mcpm_setup			@ if not, set up the cluster

	@ Otherwise, release the first man lock and skip setup:
	mov	r0, r11
	bl	vlock_unlock
	b	mcpm_setup_complete

mcpm_setup:
	@ Control dependency implies strb not observable before previous ldrb.

	@ Signal that the cluster is being brought up:
	mov	r0, #INBOUND_COMING_UP
	strb	r0, [r8, #MCPM_SYNC_CLUSTER_INBOUND]
	dmb

	@ Any CPU trying to take the cluster into CLUSTER_GOING_DOWN from this
	@ point onwards will observe INBOUND_COMING_UP and abort.

	@ Wait for any previously-pending cluster teardown operations to abort
	@ or complete:
mcpm_teardown_wait:
	ldrb	r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
	cmp	r0, #CLUSTER_GOING_DOWN
	bne	first_man_setup
	wfe
	b	mcpm_teardown_wait

first_man_setup:
	dmb

	@ If the outbound gave up before teardown started, skip cluster setup:

	cmp	r0, #CLUSTER_UP
	beq	mcpm_setup_leave

	@ power_up_setup is now responsible for setting up the cluster:

	cmp	r7, #0
	mov	r0, #1		@ second (cluster) affinity level
	blxne	r7		@ Call power_up_setup if defined
	dmb

	mov	r0, #CLUSTER_UP
	strb	r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
	dmb

mcpm_setup_leave:
	@ Leave the cluster setup critical section:

	mov	r0, #INBOUND_NOT_COMING_UP
	strb	r0, [r8, #MCPM_SYNC_CLUSTER_INBOUND]
	dsb	st
	sev

	mov	r0, r11
	bl	vlock_unlock	@ implies DMB
	b	mcpm_setup_complete

	@ In the contended case, non-first men wait here for cluster setup
	@ to complete:
mcpm_setup_wait:
	ldrb	r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
	cmp	r0, #CLUSTER_UP
	wfene
	bne	mcpm_setup_wait
	dmb

mcpm_setup_complete:
	@ If a platform-specific CPU setup hook is needed, it is
	@ called from here.

	cmp	r7, #0
	mov	r0, #0		@ first (CPU) affinity level
	blxne	r7		@ Call power_up_setup if defined
	dmb

	@ Mark the CPU as up:

	mov	r0, #CPU_UP
	strb	r0, [r5]

	@ Observability order of CPU_UP and opening of the gate does not matter.

mcpm_entry_gated:
	ldr	r5, [r6, r4, lsl #2]		@ r5 = CPU entry vector
	cmp	r5, #0
	wfeeq
	beq	mcpm_entry_gated
	dmb

	pr_dbg	"released\n"
	bx	r5

	.align	2

3:	.word	mcpm_entry_early_pokes - .
	.word	mcpm_entry_vectors - 3b
	.word	mcpm_power_up_setup_phys - 3b
	.word	mcpm_sync - 3b
	.word	first_man_locks - 3b

ENDPROC(mcpm_entry_point)

	.bss

	.align	CACHE_WRITEBACK_ORDER
	.type	first_man_locks, #object
first_man_locks:
	.space	VLOCK_SIZE * MAX_NR_CLUSTERS
	.align	CACHE_WRITEBACK_ORDER

	.type	mcpm_entry_vectors, #object
ENTRY(mcpm_entry_vectors)
	.space	4 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER

	.type	mcpm_entry_early_pokes, #object
ENTRY(mcpm_entry_early_pokes)
	.space	8 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER

	.type	mcpm_power_up_setup_phys, #object
ENTRY(mcpm_power_up_setup_phys)
	.space  4		@ set by mcpm_sync_init()