読者です 読者をやめる 読者になる 読者になる

minix2のsmp実装をminix3に移植してみるテスト。

minix


まだ、enable_cpu()で必要な処理を実装しているだけでBSPしか見えてないです。
qemuの画面が小さいからアレですが、画面の上のほうにkprintf()でデバッグ用のメッセージ出力させてます。
ハックに使ってるminixのバージョンは3.1.0ですが、smp対応ということで3.1.1と変更しました。

現状のdiffはこちら。

diff -urpN orig/usr/src/include/minix/com.h smp/usr/src/include/minix/com.h
--- orig/usr/src/include/minix/com.h	2009-08-05 23:47:30.000000000 +0900
+++ smp/usr/src/include/minix/com.h	2009-09-22 00:32:29.000000000 +0900
@@ -21,7 +21,7 @@
  */
 
 /* Kernel tasks. These all run in the same address space. */
-#define IDLE             -4	/* runs when no one else can run */
+#define IDLE(cpu)        (-4 - (cpu))	/* runs when no one else can run */
 #define CLOCK  		 -3	/* alarms and other clock functions */
 #define SYSTEM           -2	/* request system functionality */
 #define KERNEL           -1	/* pseudo-process for IPC and scheduling */
diff -urpN orig/usr/src/include/minix/config.h smp/usr/src/include/minix/config.h
--- orig/usr/src/include/minix/config.h	2009-08-05 23:47:30.000000000 +0900
+++ smp/usr/src/include/minix/config.h	2009-09-22 00:34:25.000000000 +0900
@@ -3,7 +3,7 @@
 
 /* Minix release and version numbers. */
 #define OS_RELEASE "3"
-#define OS_VERSION "1.0"
+#define OS_VERSION "1.1"
 
 /* This file sets configuration parameters for the MINIX kernel, FS, and PM.
  * It is divided up into two main sections.  The first section contains
@@ -123,4 +123,23 @@
 #define ASKDEV _ASKDEV
 #define FASTLOAD _FASTLOAD
 
+/* Enable or disable multiprocessor suport (only for intel architecture), 
+   and configure number of processor (1 cpu implicit if MP not enabled) */
+#define ENABLE_MP 1 /* 1 enable, 0 disable */
+#define MP_NR_CPUS 2 /* ignored if ENABLE_MP is 0 */
+
+/* MP version, only for printing */
+#define MP_RELEASE "1"
+#define MP_VERSION "0"
+
+/* Multiprocessor verifications */
+#if (CHIP != INTEL) && (ENABLE_MP == 1)
+error "ENABLE_MP: multiprocessor only avaliable for intel architecture"
+#endif
+
+#if (ENABLE_MP != 1) /* only 1 cpu if multiprocessor not enabled */
+#undef  MP_NR_CPUS
+#define MP_NR_CPUS 1
+#endif
+
 #endif /* _CONFIG_H */
diff -urpN orig/usr/src/kernel/Makefile smp/usr/src/kernel/Makefile
--- orig/usr/src/kernel/Makefile	2009-08-05 23:47:30.000000000 +0900
+++ smp/usr/src/kernel/Makefile	2009-09-22 00:34:59.000000000 +0900
@@ -15,7 +15,9 @@ LDFLAGS = -i 
 
 HEAD =	mpx.o
 OBJS =	start.o protect.o klib.o table.o main.o proc.o \
-	i8259.o exception.o system.o clock.o utility.o debug.o
+	i8259.o exception.o system.o clock.o utility.o debug.o \
+	mp.o xmp.o
+
 SYSTEM = system.a
 LIBS = -ltimers 
 
diff -urpN orig/usr/src/kernel/main.c smp/usr/src/kernel/main.c
--- orig/usr/src/kernel/main.c	2009-08-05 23:47:30.000000000 +0900
+++ smp/usr/src/kernel/main.c	2009-09-22 00:36:54.000000000 +0900
@@ -21,6 +21,10 @@
 #include <minix/com.h>
 #include "proc.h"
 
+/* For multi processors */ 
+#include "mp.h"
+#include "xmp.h"
+
 /* Prototype declarations for PRIVATE functions. */
 FORWARD _PROTOTYPE( void announce, (void));	
 FORWARD _PROTOTYPE( void shutdown, (timer_t *tp));
@@ -158,10 +162,18 @@ PUBLIC void main()
   /* We're definitely not shutting down. */
   shutdown_started = 0;
 
+  /* At first, we should disable all cpus state. */
+  disable_all_cpus();
+
+  /* Then, We're going to enable current cpu. */
+  enable_cpu(this_cpu);
+
   /* MINIX is now ready. All boot image processes are on the ready queue.
    * Return to the assembly code to start running the current process. 
    */
-  bill_ptr = proc_addr(IDLE);		/* it has to point somewhere */
+  /* TODO: it might be ok to point cpu0 */
+  bill_ptr = proc_addr(IDLE(0));/* it has to point somewhere */
+
   announce();				/* print MINIX startup banner */
   restart();
 }
diff -urpN orig/usr/src/kernel/mp.c smp/usr/src/kernel/mp.c
--- orig/usr/src/kernel/mp.c	1970-01-01 09:00:00.000000000 +0900
+++ smp/usr/src/kernel/mp.c	2009-09-22 15:13:24.000000000 +0900
@@ -0,0 +1,156 @@
+#include <minix/config.h>    /* for mp configuration */
+#include <minix/com.h>
+
+#include "kernel.h"
+#include "proc.h"
+
+#include "mp.h"
+
+#if 1
+#define mp_kprintf(x) kprintf x
+#else
+#define mp_kprintf(x) 
+#endif
+
+struct mp_commands {
+  int mp_command;     /* command for cpu to execute in IPI */
+  int mp_param;       /* paremeter of IPI command */
+} mp_comm[MP_NR_CPUS];
+
+/* To keep cpu's state which are Running or not */
+int cpus[MP_NR_CPUS];
+
+u32_t local_apic_base = 0x0FEE00000;
+
+#define LOCAL_APIC_ICR_LOW      0x0300  /* offset for IRC low regiter */
+#define LOCAL_APIC_ICR_HIGH     0x0310  /* offset for IRC high register */
+#define LOCAL_APIC_ESR          0x0280  /* offset for ESR register */
+#define LOCAL_APIC_SPIV         0x00F0  /* offset for SPIV register */
+#define LOCAL_APIC_TASKP        0x0080  /* offset for TASK PRIORITY register */
+#define LOCAL_APIC_LDR          0x00D0  /* offset for LD register */
+#define LOCAL_APIC_DFR          0x00E0  /* offset for DF register */
+#define LOCAL_APIC_IDR          0x0020  /* offset for ID register */
+#define LOCAL_APIC_LVTL0        0x0350  /* offset for LVT LINTIN0 */
+#define LOCAL_APIC_LVTL1        0x0360  /* offset for LVT LINTIN1 */
+#define LOCAL_APIC_EOI          0x00B0  /* offset for EOI register */
+
+#define LVT_DM_FIXED            0       /* delivery mode in L.APIC LVT */
+#define LVT_DM_NMI              4       /* delivery mode in L.APIC LVT */
+#define LVT_DM_EXTINT           7       /* delivery mode in L.APIC LVT */
+
+#define LVT_MASKED_SHIFT        16      /* shift for masked LINTINx bit */
+#define LVT_DM_SHIFT            8       /* shift for delivery mode field */
+
+#define ENABLE_APIC_SHIFT       8       /* enable APIC bit position in SPIV reg */
+
+
+#define DELIVERY_STATUS_SHIFT   12      /* status of ipi */
+#define DELIVERY_PENDING        1       /* ipi is being sent */
+#define DELIVERY_IDLE           0       /* sent: no work to be done */
+#define DELIVERY_STATUS         (LOCAL_APIC_READ(LOCAL_APIC_ICR_LOW) \
+                                              & (1<<DELIVERY_STATUS_SHIFT))
+
+#define DEST_FIELD_SHIFT (56-32)        /* destination field bits position */
+
+PRIVATE void LOCAL_APIC_WRITE(u32_t reg, u32_t val) {
+  phys_copy_dword( vir2phys(&val), local_apic_base+reg );
+}
+
+PRIVATE u32_t LOCAL_APIC_READ(u32_t reg) {
+  u32_t val;
+  phys_copy_dword( local_apic_base+reg, vir2phys(&val) );
+  return val;
+}
+
+
+PUBLIC void disable_all_cpus(void)
+{
+  int cpu;
+
+  /* All cpu should be disabled */
+  FOR_EACH_CPU(cpu) 
+    cpus[cpu] = CPU_DISABLED;
+}
+
+PUBLIC void enable_apic_interrupts(void)
+{
+  /* Enable current APIC and open to interrputs from PIC */
+  u32_t reg;
+
+  mp_kprintf(("start enable_apic_interrupts()\n"));
+
+  reg = LOCAL_APIC_READ(LOCAL_APIC_SPIV);
+  reg |= (1<<ENABLE_APIC_SHIFT);               /* Enable APIC */
+  LOCAL_APIC_WRITE(LOCAL_APIC_SPIV, reg); 
+
+  reg = LOCAL_APIC_READ(LOCAL_APIC_LVTL0);
+  reg &= ~(7<<LVT_DM_SHIFT);                   /* clear delivery mode */
+  reg &= ~(1<<LVT_MASKED_SHIFT);               /* unmask LINTIN0 */
+  reg |= (LVT_DM_EXTINT<<LVT_DM_SHIFT);        /* ExtINT at LINTINT0 */
+  LOCAL_APIC_WRITE(LOCAL_APIC_LVTL0, reg);
+  
+  reg = LOCAL_APIC_READ(LOCAL_APIC_LVTL1);
+  reg &= ~(7<<LVT_DM_SHIFT);                   /* clear delivery mode */
+  reg &= ~(1<<LVT_MASKED_SHIFT);               /* ummask LINTIN1 */
+  reg |= (LVT_DM_NMI<<LVT_DM_SHIFT);           /* NMI at LINTINT1 */
+  LOCAL_APIC_WRITE(LOCAL_APIC_LVTL1, reg);
+
+}
+
+
+PRIVATE void forward_vector_to_cpu(u8_t vector, int cpu) 
+{
+  /* Send a interrupt vector to a CPU */
+  u32_t icr_h,icr_l;
+
+  while (DELIVERY_STATUS); 
+  /* Wait for local APIC to complete previous IPI */
+
+  /* prepare to send IPI */
+  icr_h = LOCAL_APIC_READ(LOCAL_APIC_ICR_HIGH);
+  icr_l = LOCAL_APIC_READ(LOCAL_APIC_ICR_LOW);
+
+  icr_l &= ~0x000CDFFF;  /* clear non-reserved fields */
+  icr_h &= 0x00FFFFFF;  /* clear non-reserved fields */
+
+  icr_l |= vector;                             /* vector field */
+  icr_h |= (cpu << DEST_FIELD_SHIFT);          /* cpu to interrupt */
+  
+  /* send the IPI */
+  LOCAL_APIC_WRITE(LOCAL_APIC_ICR_HIGH, icr_h);
+  LOCAL_APIC_WRITE(LOCAL_APIC_ICR_LOW,  icr_l);
+
+  mp_kprintf(("Send param High is %d and Low is %d\n", icr_h, icr_l));
+}
+
+
+PUBLIC void interrupt_cpu(int cpu, int command, int param) 
+{
+  /* Send a message to a CPU consisting in a command and an optional parameter */
+
+  mp_comm[cpu].mp_command =command;
+  mp_comm[cpu].mp_param =param;
+  forward_vector_to_cpu(VECTOR(16),cpu);
+}
+
+PUBLIC void enable_cpu(int cpu)
+{
+  mp_kprintf(("current cpu is %d\n", cpu));
+
+  /* if cpu is enable, it won't do anything. */
+  if (cpus[cpu] != CPU_ENABLED) {
+    mp_kprintf(("cpus[%d] is Disabled\n", cpu));
+    if (cpu == this_cpu) {
+      mp_kprintf(("cpu(%d) is this_cpu\n", cpu));
+
+      enable_apic_interrupts();
+      cpus[cpu] = CPU_ENABLED;
+      /*lock_pick_proc(); */
+      kprintf("CPU%d enabled\n",cpu);
+    } else {
+      interrupt_cpu(cpu, MP_CM_ENABLE, WITH_ECHO);
+    }
+  }
+
+
+}
diff -urpN orig/usr/src/kernel/mp.h smp/usr/src/kernel/mp.h
--- orig/usr/src/kernel/mp.h	1970-01-01 09:00:00.000000000 +0900
+++ smp/usr/src/kernel/mp.h	2009-09-22 13:16:35.000000000 +0900
@@ -0,0 +1,55 @@
+#ifndef __MP_H
+#define __MP_H
+
+#include <sys/types.h>
+
+/* These states describes cpu's state */
+#define CPU_ENABLED     1               /* cpu is runing */
+#define CPU_DISABLED    0               /* cpu is not runing */
+#define CPU_DISABLING   2               /* cpu is ordered to disable */
+#define CPU_HALTED      3               /* cpu is halted */
+
+/* this_cpu points to current cpu number. */
+#if (ENABLE_MP == 1) 
+#define this_cpu         get_current_cpu()
+#else
+#define this_cpu         0
+#endif
+
+/* MP interprocessor communication */
+#define MP_CM_NONE      0       /* nothig to do */
+#define MP_CM_DISABLE   1       /* disable cpu */
+#define MP_CM_ENABLE    2       /* enable cpu */
+#define MP_CM_SCHED     3       /* schedule and pick a new process to run */
+#define MP_CM_PICKPROC  4       /* pick a new process to run */
+#define MP_CM_REBOOT    5       /* invoke wreboot(mp_param) */
+#define MP_CM_HALT      6       /* stop cpu */
+#define MP_CM_ARG_NONE  0       /* no parameter */
+
+#define WITH_ECHO       1
+#define WITHOUT_ECHO    0
+
+/************************************************************* 
+ * PUBLIC functions for multi processors 
+ *************************************************************/
+EXTERN void disable_all_cpus(void);
+EXTERN void enable_cpu(int cpu);
+EXTERN void enable_apic_interrupts(void);
+EXTERN void interrupt_cpu(int cpu, int command, int param);
+
+/************************************************************* 
+ * PUBLIC functions for multi processors written in asm 
+ *************************************************************/
+EXTERN int get_current_cpu(void);
+EXTERN void phys_copy_dword (unsigned long source, unsigned long destination);
+EXTERN void mp_switching_lock(void);
+EXTERN void mp_switching_unlock(void);
+ 
+/* loop from cpu0 to MP_NR_CPUS */
+#define FOR_EACH_CPU(cpu) for ((cpu) = 0; (cpu) < MP_NR_CPUS; (cpu)++)
+
+/* loop all application processors */
+#define FOR_EACH_AP(cpu) for ((cpu) = 1; (cpu) < MP_NR_CPUS; (cpu)++)
+
+#endif /* __MP_H */
+  
diff -urpN orig/usr/src/kernel/proc.c smp/usr/src/kernel/proc.c
--- orig/usr/src/kernel/proc.c	2009-08-05 23:47:30.000000000 +0900
+++ smp/usr/src/kernel/proc.c	2009-09-22 13:25:23.000000000 +0900
@@ -41,6 +41,8 @@
 #include "kernel.h"
 #include "proc.h"
 
+#include "mp.h"
+
 /* Scheduling and message passing functions. The functions are available to 
  * other parts of the kernel through lock_...(). The lock temporarily disables 
  * interrupts to prevent race conditions. 
@@ -598,3 +600,16 @@ struct proc *rp;		/* this process is no 
   unlock(4);
 }
 
+/*===========================================================================*
+ *				lock_pick_proc				     *
+ *===========================================================================*/
+PUBLIC void lock_pick_proc(void)
+{
+
+  /* Safe gateway to pick_proc() for tasks. */
+
+  mp_switching_lock();
+  pick_proc();
+  mp_switching_unlock();
+
+}
diff -urpN orig/usr/src/kernel/proc.h smp/usr/src/kernel/proc.h
--- orig/usr/src/kernel/proc.h	2009-08-05 23:47:30.000000000 +0900
+++ smp/usr/src/kernel/proc.h	2009-09-22 02:00:23.000000000 +0900
@@ -107,4 +107,6 @@ EXTERN struct proc *pproc_addr[NR_TASKS 
 EXTERN struct proc *rdy_head[NR_SCHED_QUEUES]; /* ptrs to ready list headers */
 EXTERN struct proc *rdy_tail[NR_SCHED_QUEUES]; /* ptrs to ready list tails */
 
+/* for MP */
+EXTERN void lock_pick_proc(void);
 #endif /* PROC_H */
diff -urpN orig/usr/src/kernel/table.c smp/usr/src/kernel/table.c
--- orig/usr/src/kernel/table.c	2009-08-05 23:47:30.000000000 +0900
+++ smp/usr/src/kernel/table.c	2009-09-22 00:39:51.000000000 +0900
@@ -95,7 +95,7 @@ PUBLIC char *t_stack[TOT_STACK_SPACE / s
  */
 PUBLIC struct boot_image image[] = {
 /* process nr,   pc, flags, qs,  queue, stack, traps, ipcto, call,  name */ 
- { IDLE,  idle_task, IDL_F,  8, IDLE_Q, IDL_S,     0,     0,     0, "IDLE"  },
+ { IDLE(0),  idle_task, IDL_F,  8, IDLE_Q, IDL_S,     0,     0,     0, "IDLE"  },
  { CLOCK,clock_task, TSK_F, 64, TASK_Q, TSK_S, TSK_T,     0,     0, "CLOCK" },
  { SYSTEM, sys_task, TSK_F, 64, TASK_Q, TSK_S, TSK_T,     0,     0, "SYSTEM"},
  { HARDWARE,      0, TSK_F, 64, TASK_Q, HRD_S,     0,     0,     0, "KERNEL"},
diff -urpN orig/usr/src/kernel/xmp.h smp/usr/src/kernel/xmp.h
--- orig/usr/src/kernel/xmp.h	1970-01-01 09:00:00.000000000 +0900
+++ smp/usr/src/kernel/xmp.h	2009-09-22 13:09:13.000000000 +0900
@@ -0,0 +1,36 @@
+#ifndef __XMP_H
+#define __XMP_H
+
+#include <minix/config.h>
+#include <minix/const.h>
+#include "const.h"
+
+/* This code defines semaphores safe from parallel access
+   To lock we need a semaphore variable and a label id (unique)
+   To unlock we need the semaphore variable
+   A semaphore variable is a data (>=1byte) dobleword aligned */
+#define MP_LOCK_FREEVAL         0 
+
+#if ( ENABLE_MP == 1 ) 
+
+#define MP_LOCK(semaphore_var)                          ;\
+0:              clc                                     ;\
+        lock    bts     (semaphore_var),        0       ;\
+1:              cmp     (semaphore_var),MP_LOCK_FREEVAL ;\
+                jne     1b                              ;\
+                jmp     0b                              ;\
+2:              
+
+#define MP_UNLOCK(semaphore_var)                        ;\
+        lock    btr     (semaphore_var),        0       
+
+#else
+#define MP_LOCK(semaphore_var)                          ;\
+                mov     (semaphore_var),        1
+
+#define MP_UNLOCK(semaphore_var)                        ;\
+                mov     (semaphore_var),        0       
+
+#endif
+
+#endif /* __XMP_H */
diff -urpN orig/usr/src/kernel/xmp.s smp/usr/src/kernel/xmp.s
--- orig/usr/src/kernel/xmp.s	1970-01-01 09:00:00.000000000 +0900
+++ smp/usr/src/kernel/xmp.s	2009-09-22 13:12:08.000000000 +0900
@@ -0,0 +1,68 @@
+#include "xmp.h"
+#include "protect.h"
+
+.define _get_current_cpu
+.define _phys_copy_dword
+.define _mp_switching_lock
+.define _mp_switching_unlock	
+	
+.sect   .text;
+.align 4
+
+! Returns APIC id of curret cpu (0..n-1)
+_get_current_cpu:		
+        o16     push    ds
+	        push    edx
+		mov     edx,    (_local_apic_base)
+		add     edx,    0x20
+		mov     eax,    FLAT_DS_SELECTOR
+		mov     ds,     eax
+		mov     eax,    (edx)
+		and     eax,    0x0F000000 
+		shr     eax,    6*4
+                pop     edx
+        o16     pop     ds
+                ret
+
+
+! PUBLIC void phys_copy_dword(phys_bytes source, phys_bytes destination) ;
+! Copy a block of physical memory
+PC_ARGS =       4 + 4 + 4 + 4   ! 4 + 4 
+!               es edi esi eip   src dst
+.align  16
+_phys_copy_dword:	
+        cld
+        push    esi
+        push    edi
+        push    ds
+	
+	mov     eax, FLAT_DS_SELECTOR
+	mov     ds, ax
+
+        mov     eax, PC_ARGS(esp)       ! src
+	mov     ebx, PC_ARGS+4(esp)     ! dst
+
+        mov     edx, (eax)
+	mov     (ebx), edx
+	
+	pop     ds
+	pop     edi
+	pop     esi
+	ret
+
+! locks a multiprocessor-safe semaphore for switching jobs
+.sect   .data
+.align  4
+switching_data:	 .data4  MP_LOCK_FREEVAL
+.sect   .text
+	
+_mp_switching_lock:	
+        MP_LOCK(switching_data)
+        ret
+
+! unlocks a multiprocessor-safe semaphore for switching jobs
+_mp_switching_unlock:	
+        MP_UNLOCK(switching_data)
+        ret
+
+				
diff -urpN orig/usr/src/servers/is/dmp_kernel.c smp/usr/src/servers/is/dmp_kernel.c
--- orig/usr/src/servers/is/dmp_kernel.c	2009-09-22 00:19:48.000000000 +0900
+++ smp/usr/src/servers/is/dmp_kernel.c	2009-09-22 01:18:41.000000000 +0900
@@ -354,7 +354,8 @@ PUBLIC void privileges_dmp()
   for (rp = oldrp; rp < END_PROC_ADDR; rp++) {
 	if (isemptyp(rp)) continue;
 	if (++n > 23) break;
-	if (proc_nr(rp) == IDLE) 	printf("(%2d) ", proc_nr(rp));  
+	/* TODO: cpu number should get via this_cpu */
+	if (proc_nr(rp) == IDLE(0)) 	printf("(%2d) ", proc_nr(rp));  
 	else if (proc_nr(rp) < 0) 	printf("[%2d] ", proc_nr(rp));
 	else 				printf(" %2d  ", proc_nr(rp));
         r = -1;
@@ -413,7 +414,8 @@ PUBLIC void sendmask_dmp()
         if (++n > 20) break;
 
     	printf("%8s ", rp->p_name);
-	if (proc_nr(rp) == IDLE) 	printf("(%2d) ", proc_nr(rp));  
+	/* TODO: cpu number should get via this_cpu */
+	if (proc_nr(rp) == IDLE(0)) 	printf("(%2d) ", proc_nr(rp));  
 	else if (proc_nr(rp) < 0) 	printf("[%2d] ", proc_nr(rp));
 	else 				printf(" %2d  ", proc_nr(rp));
 
@@ -471,7 +473,8 @@ PUBLIC void proctab_dmp()
 	data = rp->p_memmap[D].mem_phys;
 	size = rp->p_memmap[T].mem_len
 		+ ((rp->p_memmap[S].mem_phys + rp->p_memmap[S].mem_len) - data);
-	if (proc_nr(rp) == IDLE) 	printf("(%2d) ", proc_nr(rp));  
+	/* TODO: cpu number should get via this_cpu */
+	if (proc_nr(rp) == IDLE(0)) 	printf("(%2d) ", proc_nr(rp));  
 	else if (proc_nr(rp) < 0) 	printf("[%2d] ", proc_nr(rp));
 	else 				printf(" %2d  ", proc_nr(rp));
 	printf(" %-8.8s %02u/%02u %02u/%02u %6lu%6lu %6uK%6uK%6uK %s",