SGI UV: TLB shootdown using broadcast assist unit

TLB shootdown for SGI UV.

v5: 6/12 corrections/improvements per Ingo's second review

Signed-off-by: Cliff Wickman <cpw@sgi.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

1 files changed, 85 insertions, 98 deletions

diff --git a/arch/x86/kernel/tlb_uv.c b/arch/x86/kernel/tlb_uv.c
index f7bc6a6..d8705e9 100644
--- a/arch/x86/kernel/tlb_uv.c
+++ b/arch/x86/kernel/tlb_uv.c
@@ -25,15 +25,8 @@ static int uv_bau_retry_limit __read_mostly;
 static int uv_nshift __read_mostly; /* position of pnode (which is nasid>>1) */
 static unsigned long uv_mmask __read_mostly;
 
-char *status_table[] = {
-	"IDLE",
-	"ACTIVE",
-	"DESTINATION TIMEOUT",
-	"SOURCE TIMEOUT"
-};
-
-DEFINE_PER_CPU(struct ptc_stats, ptcstats);
-DEFINE_PER_CPU(struct bau_control, bau_control);
+static DEFINE_PER_CPU(struct ptc_stats, ptcstats);
+static DEFINE_PER_CPU(struct bau_control, bau_control);
 
 /*
  * Free a software acknowledge hardware resource by clearing its Pending
@@ -55,7 +48,6 @@ static void uv_reply_to_message(int resource,
 	if (msp)
 		msp->seen_by.bits = 0;
 	uv_write_local_mmr(UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS, dw);
-	return;
 }
 
 /*
@@ -73,7 +65,7 @@ static void uv_bau_process_message(struct bau_payload_queue_entry *msg,
 	cpu = uv_blade_processor_id();
 	msg->number_of_cpus =
 	    uv_blade_nr_online_cpus(uv_node_to_blade_id(numa_node_id()));
-	this_cpu_mask = (unsigned long)1 << cpu;
+	this_cpu_mask = 1UL << cpu;
 	if (msp->seen_by.bits & this_cpu_mask)
 		return;
 	atomic_or_long(&msp->seen_by.bits, this_cpu_mask);
@@ -94,53 +86,60 @@ static void uv_bau_process_message(struct bau_payload_queue_entry *msg,
 	atomic_inc_short(&msg->acknowledge_count);
 	if (msg->number_of_cpus == msg->acknowledge_count)
 		uv_reply_to_message(sw_ack_slot, msg, msp);
-	return;
 }
 
 /*
- * Examine the payload queue on all the distribution nodes to see
+ * Examine the payload queue on one distribution node to see
  * which messages have not been seen, and which cpu(s) have not seen them.
  *
  * Returns the number of cpu's that have not responded.
  */
-static int uv_examine_destinations(struct bau_target_nodemask *distribution)
+static int uv_examine_destination(struct bau_control *bau_tablesp, int sender)
 {
-	int sender;
 	int i;
 	int j;
-	int k;
 	int count = 0;
-	struct bau_control *bau_tablesp;
 	struct bau_payload_queue_entry *msg;
 	struct bau_msg_status *msp;
 
+	for (msg = bau_tablesp->va_queue_first, i = 0; i < DEST_Q_SIZE;
+	     msg++, i++) {
+		if ((msg->sending_cpu == sender) && (!msg->replied_to)) {
+			msp = bau_tablesp->msg_statuses + i;
+			printk(KERN_DEBUG
+			       "blade %d: address:%#lx %d of %d, not cpu(s): ",
+			       i, msg->address, msg->acknowledge_count,
+			       msg->number_of_cpus);
+			for (j = 0; j < msg->number_of_cpus; j++) {
+				if (!((long)1 << j & msp-> seen_by.bits)) {
+					count++;
+					printk("%d ", j);
+				}
+			}
+			printk("\n");
+		}
+	}
+	return count;
+}
+
+/*
+ * Examine the payload queue on all the distribution nodes to see
+ * which messages have not been seen, and which cpu(s) have not seen them.
+ *
+ * Returns the number of cpu's that have not responded.
+ */
+static int uv_examine_destinations(struct bau_target_nodemask *distribution)
+{
+	int sender;
+	int i;
+	int count = 0;
+
 	sender = smp_processor_id();
 	for (i = 0; i < (sizeof(struct bau_target_nodemask) * BITSPERBYTE);
 	     i++) {
 		if (!bau_node_isset(i, distribution))
 			continue;
-		bau_tablesp = uv_bau_table_bases[i];
-		for (msg = bau_tablesp->va_queue_first, j = 0;
-		     j < DESTINATION_PAYLOAD_QUEUE_SIZE; msg++, j++) {
-			if ((msg->sending_cpu == sender) &&
-			    (!msg->replied_to)) {
-				msp = bau_tablesp->msg_statuses + j;
-				printk(KERN_DEBUG
-				"blade %d: address:%#lx %d of %d, not cpu(s): ",
-				       i, msg->address,
-				       msg->acknowledge_count,
-				       msg->number_of_cpus);
-				for (k = 0; k < msg->number_of_cpus;
-				     k++) {
-					if (!((long)1 << k & msp->
-					      seen_by.bits)) {
-						count++;
-						printk("%d ", k);
-					}
-				}
-				printk("\n");
-			}
-		}
+		count += uv_examine_destination(uv_bau_table_bases[i], sender);
 	}
 	return count;
 }
@@ -150,7 +149,7 @@ static int uv_examine_destinations(struct bau_target_nodemask *distribution)
  *
  * return COMPLETE, RETRY or GIVEUP
  */
-static int uv_wait_completion(struct bau_activation_descriptor *bau_desc,
+static int uv_wait_completion(struct bau_desc *bau_desc,
 			      unsigned long mmr_offset, int right_shift)
 {
 	int exams = 0;
@@ -213,8 +212,8 @@ static int uv_wait_completion(struct bau_activation_descriptor *bau_desc,
  * Returns 0 if some remote flushing remains to be done. The mask is left
  * unchanged.
  */
-int uv_flush_send_and_wait(int cpu, int this_blade,
-	struct bau_activation_descriptor *bau_desc, cpumask_t *cpumaskp)
+int uv_flush_send_and_wait(int cpu, int this_blade, struct bau_desc *bau_desc,
+			   cpumask_t *cpumaskp)
 {
 	int completion_status = 0;
 	int right_shift;
@@ -237,8 +236,8 @@ int uv_flush_send_and_wait(int cpu, int this_blade,
 	time1 = get_cycles();
 	do {
 		tries++;
-		index = ((unsigned long)
-			1 << UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_SHFT) | cpu;
+		index = (1UL << UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_SHFT) |
+			cpu;
 		uv_write_local_mmr(UVH_LB_BAU_SB_ACTIVATION_CONTROL, index);
 		completion_status = uv_wait_completion(bau_desc, mmr_offset,
 					right_shift);
@@ -303,7 +302,7 @@ int uv_flush_tlb_others(cpumask_t *cpumaskp, struct mm_struct *mm,
 	int cpu;
 	int this_blade;
 	int locals = 0;
-	struct bau_activation_descriptor *bau_desc;
+	struct bau_desc *bau_desc;
 
 	cpu = uv_blade_processor_id();
 	this_blade = uv_numa_blade_id();
@@ -315,8 +314,7 @@ int uv_flush_tlb_others(cpumask_t *cpumaskp, struct mm_struct *mm,
 	i = 0;
 	for_each_cpu_mask(bit, *cpumaskp) {
 		blade = uv_cpu_to_blade_id(bit);
-		if (blade > (UV_DISTRIBUTION_SIZE - 1))
-			BUG();
+		BUG_ON(blade > (UV_DISTRIBUTION_SIZE - 1));
 		if (blade == this_blade) {
 			locals++;
 			continue;
@@ -360,6 +358,8 @@ void uv_bau_message_interrupt(struct pt_regs *regs)
 {
 	struct bau_payload_queue_entry *pqp;
 	struct bau_payload_queue_entry *msg;
+	struct bau_payload_queue_entry *va_queue_first;
+	struct bau_payload_queue_entry *va_queue_last;
 	struct pt_regs *old_regs = set_irq_regs(regs);
 	cycles_t time1, time2;
 	int msg_slot;
@@ -376,7 +376,8 @@ void uv_bau_message_interrupt(struct pt_regs *regs)
 
 	local_pnode = uv_blade_to_pnode(uv_numa_blade_id());
 
-	pqp = __get_cpu_var(bau_control).va_queue_first;
+	pqp = va_queue_first = __get_cpu_var(bau_control).va_queue_first;
+	va_queue_last = __get_cpu_var(bau_control).va_queue_last;
 	msg = __get_cpu_var(bau_control).bau_msg_head;
 	while (msg->sw_ack_vector) {
 		count++;
@@ -387,8 +388,8 @@ void uv_bau_message_interrupt(struct pt_regs *regs)
 		uv_bau_process_message(msg, msg_slot, sw_ack_slot);
 
 		msg++;
-		if (msg > __get_cpu_var(bau_control).va_queue_last)
-			msg = __get_cpu_var(bau_control).va_queue_first;
+		if (msg > va_queue_last)
+			msg = va_queue_first;
 		__get_cpu_var(bau_control).bau_msg_head = msg;
 	}
 	if (!count)
@@ -401,7 +402,6 @@ void uv_bau_message_interrupt(struct pt_regs *regs)
 
 	irq_exit();
 	set_irq_regs(old_regs);
-	return;
 }
 
 static void uv_enable_timeouts(void)
@@ -423,7 +423,6 @@ static void uv_enable_timeouts(void)
 		pnode = uv_blade_to_pnode(blade);
 		cur_cpu += uv_blade_nr_possible_cpus(i);
 	}
-	return;
 }
 
 static void *uv_ptc_seq_start(struct seq_file *file, loff_t *offset)
@@ -535,10 +534,10 @@ static ssize_t uv_ptc_proc_write(struct file *file, const char __user *user,
 }
 
 static const struct seq_operations uv_ptc_seq_ops = {
-	.start	= uv_ptc_seq_start,
-	.next	= uv_ptc_seq_next,
-	.stop	= uv_ptc_seq_stop,
-	.show	= uv_ptc_seq_show
+	.start		= uv_ptc_seq_start,
+	.next		= uv_ptc_seq_next,
+	.stop		= uv_ptc_seq_stop,
+	.show		= uv_ptc_seq_show
 };
 
 static int uv_ptc_proc_open(struct inode *inode, struct file *file)
@@ -568,6 +567,7 @@ static int __init uv_ptc_init(void)
 	if (!proc_uv_ptc) {
 		printk(KERN_ERR "unable to create %s proc entry\n",
 		       UV_PTC_BASENAME);
+		remove_proc_entry("sgi_uv", NULL);
 		return -EINVAL;
 	}
 	proc_uv_ptc->proc_fops = &proc_uv_ptc_operations;
@@ -582,33 +582,26 @@ static struct bau_control * __init uv_table_bases_init(int blade, int node)
 	int i;
 	int *ip;
 	struct bau_msg_status *msp;
-	struct bau_control *bau_tablesp;
+	struct bau_control *bau_tabp;
 
-	bau_tablesp =
+	bau_tabp =
 	    kmalloc_node(sizeof(struct bau_control), GFP_KERNEL, node);
-	if (!bau_tablesp)
-		BUG();
-	bau_tablesp->msg_statuses =
+	BUG_ON(!bau_tabp);
+	bau_tabp->msg_statuses =
 	    kmalloc_node(sizeof(struct bau_msg_status) *
-			 DESTINATION_PAYLOAD_QUEUE_SIZE, GFP_KERNEL, node);
-	if (!bau_tablesp->msg_statuses)
-		BUG();
-	for (i = 0, msp = bau_tablesp->msg_statuses;
-	     i < DESTINATION_PAYLOAD_QUEUE_SIZE; i++, msp++) {
+			 DEST_Q_SIZE, GFP_KERNEL, node);
+	BUG_ON(!bau_tabp->msg_statuses);
+	for (i = 0, msp = bau_tabp->msg_statuses; i < DEST_Q_SIZE; i++, msp++)
 		bau_cpubits_clear(&msp->seen_by, (int)
 				  uv_blade_nr_possible_cpus(blade));
-	}
-	bau_tablesp->watching =
-	    kmalloc_node(sizeof(int) * DESTINATION_NUM_RESOURCES,
-			 GFP_KERNEL, node);
-	if (!bau_tablesp->watching)
-		BUG();
-	for (i = 0, ip = bau_tablesp->watching;
-	     i < DESTINATION_PAYLOAD_QUEUE_SIZE; i++, ip++) {
+	bau_tabp->watching =
+	    kmalloc_node(sizeof(int) * DEST_NUM_RESOURCES, GFP_KERNEL, node);
+	BUG_ON(!bau_tabp->watching);
+	for (i = 0, ip = bau_tabp->watching; i < DEST_Q_SIZE; i++, ip++) {
 		*ip = 0;
 	}
-	uv_bau_table_bases[blade] = bau_tablesp;
-	return bau_tablesp;
+	uv_bau_table_bases[blade] = bau_tabp;
+	return bau_tabsp;
 }
 
 /*
@@ -616,7 +609,7 @@ static struct bau_control * __init uv_table_bases_init(int blade, int node)
  */
 static void __init uv_table_bases_finish(int blade, int node, int cur_cpu,
 				  struct bau_control *bau_tablesp,
-				  struct bau_activation_descriptor *adp)
+				  struct bau_desc *adp)
 {
 	int i;
 	struct bau_control *bcp;
@@ -636,7 +629,7 @@ static void __init uv_table_bases_finish(int blade, int node, int cur_cpu,
 /*
  * initialize the sending side's sending buffers
  */
-static struct bau_activation_descriptor * __init
+static struct bau_desc * __init
 uv_activation_descriptor_init(int node, int pnode)
 {
 	int i;
@@ -644,13 +637,12 @@ uv_activation_descriptor_init(int node, int pnode)
 	unsigned long m;
 	unsigned long n;
 	unsigned long mmr_image;
-	struct bau_activation_descriptor *adp;
-	struct bau_activation_descriptor *ad2;
+	struct bau_desc *adp;
+	struct bau_desc *ad2;
 
-	adp = (struct bau_activation_descriptor *)
+	adp = (struct bau_desc *)
 	    kmalloc_node(16384, GFP_KERNEL, node);
-	if (!adp)
-		BUG();
+	BUG_ON(!adp);
 	pa = __pa((unsigned long)adp);
 	n = pa >> uv_nshift;
 	m = pa & uv_mmask;
@@ -660,7 +652,7 @@ uv_activation_descriptor_init(int node, int pnode)
 				      UVH_LB_BAU_SB_DESCRIPTOR_BASE,
 				      (n << UV_DESC_BASE_PNODE_SHIFT | m));
 	for (i = 0, ad2 = adp; i < UV_ACTIVATION_DESCRIPTOR_SIZE; i++, ad2++) {
-		memset(ad2, 0, sizeof(struct bau_activation_descriptor));
+		memset(ad2, 0, sizeof(struct bau_desc));
 		ad2->header.sw_ack_flag = 1;
 		ad2->header.base_dest_nodeid =
 		    uv_blade_to_pnode(uv_cpu_to_blade_id(0));
@@ -683,12 +675,10 @@ static struct bau_payload_queue_entry * __init uv_payload_queue_init(int node,
 	char *cp;
 	struct bau_payload_queue_entry *pqp;
 
-	pqp = (struct bau_payload_queue_entry *)
-	    kmalloc_node((DESTINATION_PAYLOAD_QUEUE_SIZE + 1) *
-			 sizeof(struct bau_payload_queue_entry),
-			 GFP_KERNEL, node);
-	if (!pqp)
-		BUG();
+	pqp = (struct bau_payload_queue_entry *) kmalloc_node(
+		(DEST_Q_SIZE + 1) * sizeof(struct bau_payload_queue_entry),
+		GFP_KERNEL, node);
+	BUG_ON(!pqp);
 	cp = (char *)pqp + 31;
 	pqp = (struct bau_payload_queue_entry *)(((unsigned long)cp >> 5) << 5);
 	bau_tablesp->va_queue_first = pqp;
@@ -699,13 +689,11 @@ static struct bau_payload_queue_entry * __init uv_payload_queue_init(int node,
 			      uv_physnodeaddr(pqp));
 	uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL,
 			      uv_physnodeaddr(pqp));
-	bau_tablesp->va_queue_last =
-	    pqp + (DESTINATION_PAYLOAD_QUEUE_SIZE - 1);
+	bau_tablesp->va_queue_last = pqp + (DEST_Q_SIZE - 1);
 	uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST,
 			      (unsigned long)
 			      uv_physnodeaddr(bau_tablesp->va_queue_last));
-	memset(pqp, 0, sizeof(struct bau_payload_queue_entry) *
-	       DESTINATION_PAYLOAD_QUEUE_SIZE);
+	memset(pqp, 0, sizeof(struct bau_payload_queue_entry) * DEST_Q_SIZE);
 	return pqp;
 }
 
@@ -717,7 +705,7 @@ static int __init uv_init_blade(int blade, int node, int cur_cpu)
 	int pnode;
 	unsigned long pa;
 	unsigned long apicid;
-	struct bau_activation_descriptor *adp;
+	struct bau_desc *adp;
 	struct bau_payload_queue_entry *pqp;
 	struct bau_control *bau_tablesp;
 
@@ -755,7 +743,7 @@ static int __init uv_bau_init(void)
 
 	uv_bau_retry_limit = 1;
 	uv_nshift = uv_hub_info->n_val;
-	uv_mmask = ((unsigned long)1 << uv_hub_info->n_val) - 1;
+	uv_mmask = (1UL << uv_hub_info->n_val) - 1;
 	nblades = 0;
 	last_blade = -1;
 	for_each_online_node(node) {
@@ -767,8 +755,7 @@ static int __init uv_bau_init(void)
 	}
 	uv_bau_table_bases = (struct bau_control **)
 	    kmalloc(nblades * sizeof(struct bau_control *), GFP_KERNEL);
-	if (!uv_bau_table_bases)
-		BUG();
+	BUG_ON(!uv_bau_table_bases);
 	last_blade = -1;
 	for_each_online_node(node) {
 		blade = uv_node_to_blade_id(node);