misc: mic: Update MIC host daemon with COSM changes
authorAshutosh Dixit <ashutosh.dixit@intel.com>
Wed, 30 Sep 2015 01:14:30 +0000 (18:14 -0700)
committerGreg Kroah-Hartman <gregkh@linuxfoundation.org>
Sun, 4 Oct 2015 11:54:54 +0000 (12:54 +0100)
This patch updates the MIC host daemon to work with corresponding
changes in COSM. Other MIC daemon fixes, cleanups and enhancements as
are also rolled into this patch. Changes to MIC sysfs ABI which go
into effect with this patch are also documented.

Reviewed-by: Sudeep Dutt <sudeep.dutt@intel.com>
Signed-off-by: Dasaratharaman Chandramouli <dasaratharaman.chandramouli@intel.com>
Signed-off-by: Ashutosh Dixit <ashutosh.dixit@intel.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Documentation/ABI/testing/sysfs-class-mic.txt
Documentation/mic/mic_overview.txt
Documentation/mic/mpssd/mpss
Documentation/mic/mpssd/mpssd.c
Documentation/mic/mpssd/mpssd.h
drivers/misc/mic/Makefile
drivers/misc/mic/bus/scif_bus.c
drivers/misc/mic/bus/scif_bus.h
include/uapi/linux/mic_common.h

index 13f48afc534fbf19c643702e09e308d0bfa63625..d45eed2bf12863b782103a4f16ed6199675c0660 100644 (file)
@@ -41,18 +41,15 @@ Description:
                When read, this entry provides the current state of an Intel
                MIC device in the context of the card OS. Possible values that
                will be read are:
-               "offline" - The MIC device is ready to boot the card OS. On
+               "ready" - The MIC device is ready to boot the card OS. On
                reading this entry after an OSPM resume, a "boot" has to be
                written to this entry if the card was previously shutdown
                during OSPM suspend.
-               "online" - The MIC device has initiated booting a card OS.
+               "booting" - The MIC device has initiated booting a card OS.
+               "online" - The MIC device has completed boot and is online
                "shutting_down" - The card OS is shutting down.
+               "resetting" - A reset has been initiated for the MIC device
                "reset_failed" - The MIC device has failed to reset.
-               "suspending" - The MIC device is currently being prepared for
-               suspend. On reading this entry, a "suspend" has to be written
-               to the state sysfs entry to ensure the card is shutdown during
-               OSPM suspend.
-               "suspended" - The MIC device has been suspended.
 
                When written, this sysfs entry triggers different state change
                operations depending upon the current state of the card OS.
@@ -62,8 +59,6 @@ Description:
                        sysfs entries.
                "reset" - Initiates device reset.
                "shutdown" - Initiates card OS shutdown.
-               "suspend" - Initiates card OS shutdown and also marks the card
-               as suspended.
 
 What:          /sys/class/mic/mic(x)/shutdown_status
 Date:          October 2013
@@ -126,7 +121,7 @@ Description:
                the card. This sysfs entry can be written with the following
                valid strings:
                a) linux - Boot a Linux image.
-               b) elf - Boot an elf image for flash updates.
+               b) flash - Boot an image for flash updates.
 
 What:          /sys/class/mic/mic(x)/log_buf_addr
 Date:          October 2013
@@ -155,3 +150,17 @@ Description:
                daemon to set the log buffer length address. The correct log
                buffer length address to be written can be found in the
                System.map file of the card OS.
+
+What:          /sys/class/mic/mic(x)/heartbeat_enable
+Date:          March 2015
+KernelVersion: 3.20
+Contact:       Ashutosh Dixit <ashutosh.dixit@intel.com>
+Description:
+               The MIC drivers detect and inform user space about card crashes
+               via a heartbeat mechanism (see the description of
+               shutdown_status above). User space can turn off this
+               notification by setting heartbeat_enable to 0 and enable it by
+               setting this entry to 1. If this notification is disabled it is
+               the responsibility of user space to detect card crashes via
+               alternative means such as a network ping. This setting is
+               enabled by default.
index 1a2f2c8ec59ee88c8808374092b3cb755078e7a7..73f44fc3e71569f98bf14283239af7d1f310c9d9 100644 (file)
@@ -28,6 +28,10 @@ The Symmetric Communication Interface (SCIF (pronounced as skiff)) is a
 low level communications API across PCIe currently implemented for MIC.
 More details are available at scif_overview.txt.
 
+The Coprocessor State Management (COSM) driver on the host allows for
+boot, shutdown and reset of Intel MIC devices. It communicates with a COSM
+"client" driver on the MIC cards over SCIF to perform these functions.
+
 Here is a block diagram of the various components described above. The
 virtio backends are situated on the host rather than the card given better
 single threaded performance for the host compared to MIC, the ability of
@@ -51,19 +55,20 @@ the fact that the virtio block storage backend can only be on the host.
                       |               |       | Virtio over PCIe IOCTLs  |
                       |               |       +--------------------------+
 +-----------+         |               |                   |  +-----------+
-| MIC DMA   |         |  +----------+ | +-----------+     |  | MIC DMA   |
-| Driver    |         |  |  SCIF    | | |   SCIF    |     |  | Driver    |
-+-----------+         |  +----------+ | +-----------+     |  +-----------+
-      |               |       |       |       |           |        |
-+---------------+     | +-----+-----+ | +-----+-----+     | +---------------+
-|MIC virtual Bus|     | |SCIF HW Bus| | |SCIF HW BUS|     | |MIC virtual Bus|
-+---------------+     | +-----------+ | +-----+-----+     | +---------------+
-      |               |       |       |       |           |              |
-      |   +--------------+    |       |       |    +---------------+     |
-      |   |Intel MIC     |    |       |       |    |Intel MIC      |     |
-      +---|Card Driver   +----+       |       |    |Host Driver    |     |
-          +--------------+            |       +----+---------------+-----+
-                      |               |                   |
+| MIC DMA   |         |      +------+ | +------+ +------+ |  | MIC DMA   |
+| Driver    |         |      | SCIF | | | SCIF | | COSM | |  | Driver    |
++-----------+         |      +------+ | +------+ +--+---+ |  +-----------+
+      |               |         |     |    |        |     |        |
++---------------+     |      +------+ | +--+---+ +--+---+ | +----------------+
+|MIC virtual Bus|     |      |SCIF  | | |SCIF  | | COSM | | |MIC virtual Bus |
++---------------+     |      |HW Bus| | |HW Bus| | Bus  | | +----------------+
+      |               |      +------+ | +--+---+ +------+ |              |
+      |               |         |     |       |     |     |              |
+      |   +-----------+---+     |     |       |    +---------------+     |
+      |   |Intel MIC      |     |     |       |    |Intel MIC      |     |
+      +---|Card Driver    |     |     |       |    |Host Driver    |     |
+          +------------+--------+     |       +----+---------------+-----+
+                 |                    |                   |
              +-------------------------------------------------------------+
              |                                                             |
              |                    PCIe Bus                                 |
index 582aad4811ae802844ebeb37d51cc9a1ffec68a8..09ea90931649c68263aac41af827c9e329a9460e 100755 (executable)
@@ -119,10 +119,10 @@ stop()
        # Wait for the cards to go offline
        for f in $sysfs/*
        do
-               while [ "`cat $f/state`" != "offline" ]
+               while [ "`cat $f/state`" != "ready" ]
                do
                        sleep 1
-                       echo -e "Waiting for "`basename $f`" to go offline"
+                       echo -e "Waiting for "`basename $f`" to become ready"
                done
        done
 
index 3c5c379fc29d6797d0ce17a837cbda64278f68b3..aaeafa18d99bb70bee8cf5bd1d39829b17d6d3d4 100644 (file)
@@ -43,7 +43,7 @@
 #include <linux/mic_common.h>
 #include <tools/endian.h>
 
-static void init_mic(struct mic_info *mic);
+static void *init_mic(void *arg);
 
 static FILE *logfp;
 static struct mic_info mic_list;
@@ -116,19 +116,18 @@ static struct {
                .num = htole16(MIC_VRING_ENTRIES),
        },
 #if GSO_ENABLED
-               .host_features = htole32(
+       .host_features = htole32(
                1 << VIRTIO_NET_F_CSUM |
                1 << VIRTIO_NET_F_GSO |
                1 << VIRTIO_NET_F_GUEST_TSO4 |
                1 << VIRTIO_NET_F_GUEST_TSO6 |
-               1 << VIRTIO_NET_F_GUEST_ECN |
-               1 << VIRTIO_NET_F_GUEST_UFO),
+               1 << VIRTIO_NET_F_GUEST_ECN),
 #else
                .host_features = 0,
 #endif
 };
 
-static const char *mic_config_dir = "/etc/sysconfig/mic";
+static const char *mic_config_dir = "/etc/mpss";
 static const char *virtblk_backend = "VIRTBLK_BACKEND";
 static struct {
        struct mic_device_desc dd;
@@ -192,7 +191,7 @@ tap_configure(struct mic_info *mic, char *dev)
                return ret;
        }
 
-       snprintf(ipaddr, IFNAMSIZ, "172.31.%d.254/24", mic->id);
+       snprintf(ipaddr, IFNAMSIZ, "172.31.%d.254/24", mic->id + 1);
 
        pid = fork();
        if (pid == 0) {
@@ -255,8 +254,7 @@ static int tun_alloc(struct mic_info *mic, char *dev)
                return err;
        }
 #if GSO_ENABLED
-       offload = TUN_F_CSUM | TUN_F_TSO4 | TUN_F_TSO6 |
-               TUN_F_TSO_ECN | TUN_F_UFO;
+       offload = TUN_F_CSUM | TUN_F_TSO4 | TUN_F_TSO6 | TUN_F_TSO_ECN;
 
        err = ioctl(fd, TUNSETOFFLOAD, offload);
        if (err < 0) {
@@ -332,7 +330,6 @@ static struct mic_device_desc *get_device_desc(struct mic_info *mic, int type)
                        return d;
        }
        mpsslog("%s %s %d not found\n", mic->name, __func__, type);
-       assert(0);
        return NULL;
 }
 
@@ -415,6 +412,13 @@ mic_virtio_copy(struct mic_info *mic, int fd,
        return ret;
 }
 
+static inline unsigned _vring_size(unsigned int num, unsigned long align)
+{
+       return ((sizeof(struct vring_desc) * num + sizeof(__u16) * (3 + num)
+                               + align - 1) & ~(align - 1))
+               + sizeof(__u16) * 3 + sizeof(struct vring_used_elem) * num;
+}
+
 /*
  * This initialization routine requires at least one
  * vring i.e. vr0. vr1 is optional.
@@ -426,8 +430,9 @@ init_vr(struct mic_info *mic, int fd, int type,
        int vr_size;
        char *va;
 
-       vr_size = PAGE_ALIGN(vring_size(MIC_VRING_ENTRIES,
-               MIC_VIRTIO_RING_ALIGN) + sizeof(struct _mic_vring_info));
+       vr_size = PAGE_ALIGN(_vring_size(MIC_VRING_ENTRIES,
+                                        MIC_VIRTIO_RING_ALIGN) +
+                            sizeof(struct _mic_vring_info));
        va = mmap(NULL, MIC_DEVICE_PAGE_END + vr_size * num_vq,
                PROT_READ, MAP_SHARED, fd, 0);
        if (MAP_FAILED == va) {
@@ -439,25 +444,25 @@ init_vr(struct mic_info *mic, int fd, int type,
        set_dp(mic, type, va);
        vr0->va = (struct mic_vring *)&va[MIC_DEVICE_PAGE_END];
        vr0->info = vr0->va +
-               vring_size(MIC_VRING_ENTRIES, MIC_VIRTIO_RING_ALIGN);
+               _vring_size(MIC_VRING_ENTRIES, MIC_VIRTIO_RING_ALIGN);
        vring_init(&vr0->vr,
                   MIC_VRING_ENTRIES, vr0->va, MIC_VIRTIO_RING_ALIGN);
        mpsslog("%s %s vr0 %p vr0->info %p vr_size 0x%x vring 0x%x ",
                __func__, mic->name, vr0->va, vr0->info, vr_size,
-               vring_size(MIC_VRING_ENTRIES, MIC_VIRTIO_RING_ALIGN));
+               _vring_size(MIC_VRING_ENTRIES, MIC_VIRTIO_RING_ALIGN));
        mpsslog("magic 0x%x expected 0x%x\n",
                le32toh(vr0->info->magic), MIC_MAGIC + type);
        assert(le32toh(vr0->info->magic) == MIC_MAGIC + type);
        if (vr1) {
                vr1->va = (struct mic_vring *)
                        &va[MIC_DEVICE_PAGE_END + vr_size];
-               vr1->info = vr1->va + vring_size(MIC_VRING_ENTRIES,
+               vr1->info = vr1->va + _vring_size(MIC_VRING_ENTRIES,
                        MIC_VIRTIO_RING_ALIGN);
                vring_init(&vr1->vr,
                           MIC_VRING_ENTRIES, vr1->va, MIC_VIRTIO_RING_ALIGN);
                mpsslog("%s %s vr1 %p vr1->info %p vr_size 0x%x vring 0x%x ",
                        __func__, mic->name, vr1->va, vr1->info, vr_size,
-                       vring_size(MIC_VRING_ENTRIES, MIC_VIRTIO_RING_ALIGN));
+                       _vring_size(MIC_VRING_ENTRIES, MIC_VIRTIO_RING_ALIGN));
                mpsslog("magic 0x%x expected 0x%x\n",
                        le32toh(vr1->info->magic), MIC_MAGIC + type + 1);
                assert(le32toh(vr1->info->magic) == MIC_MAGIC + type + 1);
@@ -466,16 +471,21 @@ done:
        return va;
 }
 
-static void
+static int
 wait_for_card_driver(struct mic_info *mic, int fd, int type)
 {
        struct pollfd pollfd;
        int err;
        struct mic_device_desc *desc = get_device_desc(mic, type);
+       __u8 prev_status;
 
+       if (!desc)
+               return -ENODEV;
+       prev_status = desc->status;
        pollfd.fd = fd;
        mpsslog("%s %s Waiting .... desc-> type %d status 0x%x\n",
                mic->name, __func__, type, desc->status);
+
        while (1) {
                pollfd.events = POLLIN;
                pollfd.revents = 0;
@@ -487,8 +497,13 @@ wait_for_card_driver(struct mic_info *mic, int fd, int type)
                }
 
                if (pollfd.revents) {
-                       mpsslog("%s %s Waiting... desc-> type %d status 0x%x\n",
-                               mic->name, __func__, type, desc->status);
+                       if (desc->status != prev_status) {
+                               mpsslog("%s %s Waiting... desc-> type %d "
+                                       "status 0x%x\n",
+                                       mic->name, __func__, type,
+                                       desc->status);
+                               prev_status = desc->status;
+                       }
                        if (desc->status & VIRTIO_CONFIG_S_DRIVER_OK) {
                                mpsslog("%s %s poll.revents %d\n",
                                        mic->name, __func__, pollfd.revents);
@@ -499,6 +514,7 @@ wait_for_card_driver(struct mic_info *mic, int fd, int type)
                        }
                }
        }
+       return 0;
 }
 
 /* Spin till we have some descriptors */
@@ -575,9 +591,16 @@ virtio_net(void *arg)
                                __func__, strerror(errno));
                        continue;
                }
-               if (!(desc->status & VIRTIO_CONFIG_S_DRIVER_OK))
-                       wait_for_card_driver(mic, mic->mic_net.virtio_net_fd,
-                                            VIRTIO_ID_NET);
+               if (!(desc->status & VIRTIO_CONFIG_S_DRIVER_OK)) {
+                       err = wait_for_card_driver(mic,
+                                                  mic->mic_net.virtio_net_fd,
+                                                  VIRTIO_ID_NET);
+                       if (err) {
+                               mpsslog("%s %s %d Exiting...\n",
+                                       mic->name, __func__, __LINE__);
+                               break;
+                       }
+               }
                /*
                 * Check if there is data to be read from TUN and write to
                 * virtio net fd if there is.
@@ -786,10 +809,16 @@ virtio_console(void *arg)
                                strerror(errno));
                        continue;
                }
-               if (!(desc->status & VIRTIO_CONFIG_S_DRIVER_OK))
-                       wait_for_card_driver(mic,
-                                            mic->mic_console.virtio_console_fd,
-                               VIRTIO_ID_CONSOLE);
+               if (!(desc->status & VIRTIO_CONFIG_S_DRIVER_OK)) {
+                       err = wait_for_card_driver(mic,
+                                       mic->mic_console.virtio_console_fd,
+                                       VIRTIO_ID_CONSOLE);
+                       if (err) {
+                               mpsslog("%s %s %d Exiting...\n",
+                                       mic->name, __func__, __LINE__);
+                               break;
+                       }
+               }
 
                if (console_poll[MONITOR_FD].revents & POLLIN) {
                        copy.iov = iov0;
@@ -1048,8 +1077,9 @@ stop_virtblk(struct mic_info *mic)
 {
        int vr_size, ret;
 
-       vr_size = PAGE_ALIGN(vring_size(MIC_VRING_ENTRIES,
-               MIC_VIRTIO_RING_ALIGN) + sizeof(struct _mic_vring_info));
+       vr_size = PAGE_ALIGN(_vring_size(MIC_VRING_ENTRIES,
+                                        MIC_VIRTIO_RING_ALIGN) +
+                            sizeof(struct _mic_vring_info));
        ret = munmap(mic->mic_virtblk.block_dp,
                MIC_DEVICE_PAGE_END + vr_size * virtblk_dev_page.dd.num_vq);
        if (ret < 0)
@@ -1131,6 +1161,10 @@ write_status(int fd, __u8 *status)
        return ioctl(fd, MIC_VIRTIO_COPY_DESC, &copy);
 }
 
+#ifndef VIRTIO_BLK_T_GET_ID
+#define VIRTIO_BLK_T_GET_ID    8
+#endif
+
 static void *
 virtio_block(void *arg)
 {
@@ -1297,12 +1331,7 @@ reset(struct mic_info *mic)
                mpsslog("%s: %s %d state %s\n",
                        mic->name, __func__, __LINE__, state);
 
-               /*
-                * If the shutdown was initiated by OSPM, the state stays
-                * in "suspended" which is also a valid condition for reset.
-                */
-               if ((!strcmp(state, "offline")) ||
-                   (!strcmp(state, "suspended"))) {
+               if (!strcmp(state, "ready")) {
                        free(state);
                        break;
                }
@@ -1331,34 +1360,50 @@ get_mic_shutdown_status(struct mic_info *mic, char *shutdown_status)
        assert(0);
 };
 
-static int get_mic_state(struct mic_info *mic, char *state)
+static int get_mic_state(struct mic_info *mic)
 {
-       if (!strcmp(state, "offline"))
-               return MIC_OFFLINE;
-       if (!strcmp(state, "online"))
-               return MIC_ONLINE;
-       if (!strcmp(state, "shutting_down"))
-               return MIC_SHUTTING_DOWN;
-       if (!strcmp(state, "reset_failed"))
-               return MIC_RESET_FAILED;
-       if (!strcmp(state, "suspending"))
-               return MIC_SUSPENDING;
-       if (!strcmp(state, "suspended"))
-               return MIC_SUSPENDED;
-       mpsslog("%s: BUG invalid state %s\n", mic->name, state);
-       /* Invalid state */
-       assert(0);
+       char *state = NULL;
+       enum mic_states mic_state;
+
+       while (!state) {
+               state = readsysfs(mic->name, "state");
+               sleep(1);
+       }
+       mpsslog("%s: %s %d state %s\n",
+               mic->name, __func__, __LINE__, state);
+
+       if (!strcmp(state, "ready")) {
+               mic_state = MIC_READY;
+       } else if (!strcmp(state, "booting")) {
+               mic_state = MIC_BOOTING;
+       } else if (!strcmp(state, "online")) {
+               mic_state = MIC_ONLINE;
+       } else if (!strcmp(state, "shutting_down")) {
+               mic_state = MIC_SHUTTING_DOWN;
+       } else if (!strcmp(state, "reset_failed")) {
+               mic_state = MIC_RESET_FAILED;
+       } else if (!strcmp(state, "resetting")) {
+               mic_state = MIC_RESETTING;
+       } else {
+               mpsslog("%s: BUG invalid state %s\n", mic->name, state);
+               assert(0);
+       }
+
+       free(state);
+       return mic_state;
 };
 
 static void mic_handle_shutdown(struct mic_info *mic)
 {
 #define SHUTDOWN_TIMEOUT 60
-       int i = SHUTDOWN_TIMEOUT, ret, stat = 0;
+       int i = SHUTDOWN_TIMEOUT;
        char *shutdown_status;
        while (i) {
                shutdown_status = readsysfs(mic->name, "shutdown_status");
-               if (!shutdown_status)
+               if (!shutdown_status) {
+                       sleep(1);
                        continue;
+               }
                mpsslog("%s: %s %d shutdown_status %s\n",
                        mic->name, __func__, __LINE__, shutdown_status);
                switch (get_mic_shutdown_status(mic, shutdown_status)) {
@@ -1377,94 +1422,110 @@ static void mic_handle_shutdown(struct mic_info *mic)
                i--;
        }
 reset:
-       ret = kill(mic->pid, SIGTERM);
-       mpsslog("%s: %s %d kill pid %d ret %d\n",
-               mic->name, __func__, __LINE__,
-               mic->pid, ret);
-       if (!ret) {
-               ret = waitpid(mic->pid, &stat,
-                       WIFSIGNALED(stat));
-               mpsslog("%s: %s %d waitpid ret %d pid %d\n",
-                       mic->name, __func__, __LINE__,
-                       ret, mic->pid);
-       }
-       if (ret == mic->pid)
-               reset(mic);
+       if (!i)
+               mpsslog("%s: %s %d timing out waiting for shutdown_status %s\n",
+                       mic->name, __func__, __LINE__, shutdown_status);
+       reset(mic);
 }
 
-static void *
-mic_config(void *arg)
+static int open_state_fd(struct mic_info *mic)
 {
-       struct mic_info *mic = (struct mic_info *)arg;
-       char *state = NULL;
        char pathname[PATH_MAX];
-       int fd, ret;
-       struct pollfd ufds[1];
-       char value[4096];
+       int fd;
 
        snprintf(pathname, PATH_MAX - 1, "%s/%s/%s",
                 MICSYSFSDIR, mic->name, "state");
 
        fd = open(pathname, O_RDONLY);
-       if (fd < 0) {
+       if (fd < 0)
                mpsslog("%s: opening file %s failed %s\n",
                        mic->name, pathname, strerror(errno));
-               goto error;
+       return fd;
+}
+
+static int block_till_state_change(int fd, struct mic_info *mic)
+{
+       struct pollfd ufds[1];
+       char value[PAGE_SIZE];
+       int ret;
+
+       ufds[0].fd = fd;
+       ufds[0].events = POLLERR | POLLPRI;
+       ret = poll(ufds, 1, -1);
+       if (ret < 0) {
+               mpsslog("%s: %s %d poll failed %s\n",
+                       mic->name, __func__, __LINE__, strerror(errno));
+               return ret;
+       }
+
+       ret = lseek(fd, 0, SEEK_SET);
+       if (ret < 0) {
+               mpsslog("%s: %s %d Failed to seek to 0: %s\n",
+                       mic->name, __func__, __LINE__, strerror(errno));
+               return ret;
+       }
+
+       ret = read(fd, value, sizeof(value));
+       if (ret < 0) {
+               mpsslog("%s: %s %d Failed to read sysfs entry: %s\n",
+                       mic->name, __func__, __LINE__, strerror(errno));
+               return ret;
+       }
+
+       return 0;
+}
+
+static void *
+mic_config(void *arg)
+{
+       struct mic_info *mic = (struct mic_info *)arg;
+       int fd, ret, stat = 0;
+
+       fd = open_state_fd(mic);
+       if (fd < 0) {
+               mpsslog("%s: %s %d open state fd failed %s\n",
+                       mic->name, __func__, __LINE__, strerror(errno));
+               goto exit;
        }
 
        do {
-               ret = lseek(fd, 0, SEEK_SET);
+               ret = block_till_state_change(fd, mic);
                if (ret < 0) {
-                       mpsslog("%s: Failed to seek to file start '%s': %s\n",
-                               mic->name, pathname, strerror(errno));
-                       goto close_error1;
+                       mpsslog("%s: %s %d block_till_state_change error %s\n",
+                               mic->name, __func__, __LINE__, strerror(errno));
+                       goto close_exit;
                }
-               ret = read(fd, value, sizeof(value));
-               if (ret < 0) {
-                       mpsslog("%s: Failed to read sysfs entry '%s': %s\n",
-                               mic->name, pathname, strerror(errno));
-                       goto close_error1;
-               }
-retry:
-               state = readsysfs(mic->name, "state");
-               if (!state)
-                       goto retry;
-               mpsslog("%s: %s %d state %s\n",
-                       mic->name, __func__, __LINE__, state);
-               switch (get_mic_state(mic, state)) {
+
+               switch (get_mic_state(mic)) {
                case MIC_SHUTTING_DOWN:
                        mic_handle_shutdown(mic);
-                       goto close_error;
-               case MIC_SUSPENDING:
-                       mic->boot_on_resume = 1;
-                       setsysfs(mic->name, "state", "suspend");
-                       mic_handle_shutdown(mic);
-                       goto close_error;
-               case MIC_OFFLINE:
+                       break;
+               case MIC_READY:
+               case MIC_RESET_FAILED:
+                       ret = kill(mic->pid, SIGTERM);
+                       mpsslog("%s: %s %d kill pid %d ret %d\n",
+                               mic->name, __func__, __LINE__,
+                               mic->pid, ret);
+                       if (!ret) {
+                               ret = waitpid(mic->pid, &stat,
+                                             WIFSIGNALED(stat));
+                               mpsslog("%s: %s %d waitpid ret %d pid %d\n",
+                                       mic->name, __func__, __LINE__,
+                                       ret, mic->pid);
+                       }
                        if (mic->boot_on_resume) {
                                setsysfs(mic->name, "state", "boot");
                                mic->boot_on_resume = 0;
                        }
-                       break;
+                       goto close_exit;
                default:
                        break;
                }
-               free(state);
-
-               ufds[0].fd = fd;
-               ufds[0].events = POLLERR | POLLPRI;
-               ret = poll(ufds, 1, -1);
-               if (ret < 0) {
-                       mpsslog("%s: poll failed %s\n",
-                               mic->name, strerror(errno));
-                       goto close_error1;
-               }
        } while (1);
-close_error:
-       free(state);
-close_error1:
+
+close_exit:
        close(fd);
-error:
+exit:
        init_mic(mic);
        pthread_exit(NULL);
 }
@@ -1477,15 +1538,15 @@ set_cmdline(struct mic_info *mic)
 
        len = snprintf(buffer, PATH_MAX,
                "clocksource=tsc highres=off nohz=off ");
-       len += snprintf(buffer + len, PATH_MAX - len,
+       len += snprintf(buffer + len, PATH_MAX,
                "cpufreq_on;corec6_off;pc3_off;pc6_off ");
-       len += snprintf(buffer + len, PATH_MAX - len,
+       len += snprintf(buffer + len, PATH_MAX,
                "ifcfg=static;address,172.31.%d.1;netmask,255.255.255.0",
-               mic->id);
+               mic->id + 1);
 
        setsysfs(mic->name, "cmdline", buffer);
        mpsslog("%s: Command line: \"%s\"\n", mic->name, buffer);
-       snprintf(buffer, PATH_MAX, "172.31.%d.1", mic->id);
+       snprintf(buffer, PATH_MAX, "172.31.%d.1", mic->id + 1);
        mpsslog("%s: IPADDR: \"%s\"\n", mic->name, buffer);
 }
 
@@ -1541,8 +1602,6 @@ set_log_buf_info(struct mic_info *mic)
        close(fd);
 }
 
-static void init_mic(struct mic_info *mic);
-
 static void
 change_virtblk_backend(int x, siginfo_t *siginfo, void *p)
 {
@@ -1553,8 +1612,16 @@ change_virtblk_backend(int x, siginfo_t *siginfo, void *p)
 }
 
 static void
-init_mic(struct mic_info *mic)
+set_mic_boot_params(struct mic_info *mic)
+{
+       set_log_buf_info(mic);
+       set_cmdline(mic);
+}
+
+static void *
+init_mic(void *arg)
 {
+       struct mic_info *mic = (struct mic_info *)arg;
        struct sigaction ignore = {
                .sa_flags = 0,
                .sa_handler = SIG_IGN
@@ -1564,7 +1631,7 @@ init_mic(struct mic_info *mic)
                .sa_sigaction = change_virtblk_backend,
        };
        char buffer[PATH_MAX];
-       int err;
+       int err, fd;
 
        /*
         * Currently, one virtio block device is supported for each MIC card
@@ -1577,12 +1644,38 @@ init_mic(struct mic_info *mic)
         * the MIC daemon.
         */
        sigaction(SIGUSR1, &ignore, NULL);
+retry:
+       fd = open_state_fd(mic);
+       if (fd < 0) {
+               mpsslog("%s: %s %d open state fd failed %s\n",
+                       mic->name, __func__, __LINE__, strerror(errno));
+               sleep(2);
+               goto retry;
+       }
+
+       if (mic->restart) {
+               snprintf(buffer, PATH_MAX, "boot");
+               setsysfs(mic->name, "state", buffer);
+               mpsslog("%s restarting mic %d\n",
+                       mic->name, mic->restart);
+               mic->restart = 0;
+       }
+
+       while (1) {
+               while (block_till_state_change(fd, mic)) {
+                       mpsslog("%s: %s %d block_till_state_change error %s\n",
+                               mic->name, __func__, __LINE__, strerror(errno));
+                       sleep(2);
+                       continue;
+               }
+
+               if (get_mic_state(mic) == MIC_BOOTING)
+                       break;
+       }
 
        mic->pid = fork();
        switch (mic->pid) {
        case 0:
-               set_log_buf_info(mic);
-               set_cmdline(mic);
                add_virtio_device(mic, &virtcons_dev_page.dd);
                add_virtio_device(mic, &virtnet_dev_page.dd);
                err = pthread_create(&mic->mic_console.console_thread, NULL,
@@ -1612,24 +1705,29 @@ init_mic(struct mic_info *mic)
                        mic->name, mic->id, errno);
                break;
        default:
-               if (mic->restart) {
-                       snprintf(buffer, PATH_MAX, "boot");
-                       setsysfs(mic->name, "state", buffer);
-                       mpsslog("%s restarting mic %d\n",
-                               mic->name, mic->restart);
-                       mic->restart = 0;
-               }
-               pthread_create(&mic->config_thread, NULL, mic_config, mic);
+               err = pthread_create(&mic->config_thread, NULL,
+                                    mic_config, mic);
+               if (err)
+                       mpsslog("%s mic_config pthread_create failed %s\n",
+                               mic->name, strerror(err));
        }
+
+       return NULL;
 }
 
 static void
 start_daemon(void)
 {
        struct mic_info *mic;
+       int err;
 
-       for (mic = mic_list.next; mic != NULL; mic = mic->next)
-               init_mic(mic);
+       for (mic = mic_list.next; mic; mic = mic->next) {
+               set_mic_boot_params(mic);
+               err = pthread_create(&mic->init_thread, NULL, init_mic, mic);
+               if (err)
+                       mpsslog("%s init_mic pthread_create failed %s\n",
+                               mic->name, strerror(err));
+       }
 
        while (1)
                sleep(60);
index f5f18b15d9a057cc6e8d5d1b007424da4d765c0b..8bd64944aacce33f0be119e4c9e770ae81d13c77 100644 (file)
@@ -86,6 +86,7 @@ struct mic_info {
        int             id;
        char            *name;
        pthread_t       config_thread;
+       pthread_t       init_thread;
        pid_t           pid;
        struct mic_console_info mic_console;
        struct mic_net_info     mic_net;
index a2e75f46fe4ea6f332f48c23385bd948c3313651..e288a1106738352ee10a77e5569226ff3102edfc 100644 (file)
@@ -2,5 +2,9 @@
 # Makefile - Intel MIC Linux driver.
 # Copyright(c) 2013, Intel Corporation.
 #
+obj-$(CONFIG_INTEL_MIC_HOST) += host/
+obj-$(CONFIG_INTEL_MIC_CARD) += card/
 obj-y += bus/
 obj-$(CONFIG_SCIF) += scif/
+obj-$(CONFIG_MIC_COSM) += cosm/
+obj-$(CONFIG_MIC_COSM) += cosm_client/
index 2da7ceed015db5cf43f6fec010af64c8f6057802..fd2702143022b6ad2f015c8f87137f3367614318 100644 (file)
@@ -28,7 +28,6 @@ static ssize_t device_show(struct device *d,
 
        return sprintf(buf, "0x%04x\n", dev->id.device);
 }
-
 static DEVICE_ATTR_RO(device);
 
 static ssize_t vendor_show(struct device *d,
@@ -38,7 +37,6 @@ static ssize_t vendor_show(struct device *d,
 
        return sprintf(buf, "0x%04x\n", dev->id.vendor);
 }
-
 static DEVICE_ATTR_RO(vendor);
 
 static ssize_t modalias_show(struct device *d,
@@ -49,7 +47,6 @@ static ssize_t modalias_show(struct device *d,
        return sprintf(buf, "scif:d%08Xv%08X\n",
                       dev->id.device, dev->id.vendor);
 }
-
 static DEVICE_ATTR_RO(modalias);
 
 static struct attribute *scif_dev_attrs[] = {
@@ -144,7 +141,8 @@ struct scif_hw_dev *
 scif_register_device(struct device *pdev, int id, struct dma_map_ops *dma_ops,
                     struct scif_hw_ops *hw_ops, u8 dnode, u8 snode,
                     struct mic_mw *mmio, struct mic_mw *aper, void *dp,
-                    void __iomem *rdp, struct dma_chan **chan, int num_chan)
+                    void __iomem *rdp, struct dma_chan **chan, int num_chan,
+                    bool card_rel_da)
 {
        int ret;
        struct scif_hw_dev *sdev;
@@ -171,6 +169,7 @@ scif_register_device(struct device *pdev, int id, struct dma_map_ops *dma_ops,
        dma_set_mask(&sdev->dev, DMA_BIT_MASK(64));
        sdev->dma_ch = chan;
        sdev->num_dma_ch = num_chan;
+       sdev->card_rel_da = card_rel_da;
        dev_set_name(&sdev->dev, "scif-dev%u", sdev->dnode);
        /*
         * device_register() causes the bus infrastructure to look for a
index 335a228a8236875138fb5e6135b290e6a29daa67..94f29ac608b6a87de4165dc79bbf3ed80e0371e7 100644 (file)
@@ -46,6 +46,8 @@ struct scif_hw_dev_id {
  * @rdp - Remote device page
  * @dma_ch - Array of DMA channels
  * @num_dma_ch - Number of DMA channels available
+ * @card_rel_da - Set to true if DMA addresses programmed in the DMA engine
+ *             are relative to the card point of view
  */
 struct scif_hw_dev {
        struct scif_hw_ops *hw_ops;
@@ -59,6 +61,7 @@ struct scif_hw_dev {
        void __iomem *rdp;
        struct dma_chan **dma_ch;
        int num_dma_ch;
+       bool card_rel_da;
 };
 
 /**
@@ -114,7 +117,8 @@ scif_register_device(struct device *pdev, int id,
                     struct scif_hw_ops *hw_ops, u8 dnode, u8 snode,
                     struct mic_mw *mmio, struct mic_mw *aper,
                     void *dp, void __iomem *rdp,
-                    struct dma_chan **chan, int num_chan);
+                    struct dma_chan **chan, int num_chan,
+                    bool card_rel_da);
 void scif_unregister_device(struct scif_hw_dev *sdev);
 
 static inline struct scif_hw_dev *dev_to_scif(struct device *dev)
index 302a2ced373c4fecbf6d397af9e7af79a88874d2..e9686372029df72d72ea348f612c88494afc7c06 100644 (file)
@@ -75,12 +75,7 @@ struct mic_device_ctrl {
  * struct mic_bootparam: Virtio device independent information in device page
  *
  * @magic: A magic value used by the card to ensure it can see the host
- * @c2h_shutdown_db: Card to Host shutdown doorbell set by host
- * @h2c_shutdown_db: Host to Card shutdown doorbell set by card
  * @h2c_config_db: Host to Card Virtio config doorbell set by card
- * @shutdown_status: Card shutdown status set by card
- * @shutdown_card: Set to 1 by the host when a card shutdown is initiated
- * @tot_nodes: Total number of nodes in the SCIF network
  * @node_id: Unique id of the node
  * @h2c_scif_db - Host to card SCIF doorbell set by card
  * @c2h_scif_db - Card to host SCIF doorbell set by host
@@ -89,12 +84,7 @@ struct mic_device_ctrl {
  */
 struct mic_bootparam {
        __le32 magic;
-       __s8 c2h_shutdown_db;
-       __s8 h2c_shutdown_db;
        __s8 h2c_config_db;
-       __u8 shutdown_status;
-       __u8 shutdown_card;
-       __u8 tot_nodes;
        __u8 node_id;
        __u8 h2c_scif_db;
        __u8 c2h_scif_db;
@@ -219,12 +209,12 @@ static inline unsigned mic_total_desc_size(struct mic_device_desc *desc)
  * enum mic_states - MIC states.
  */
 enum mic_states {
-       MIC_OFFLINE = 0,
+       MIC_READY = 0,
+       MIC_BOOTING,
        MIC_ONLINE,
        MIC_SHUTTING_DOWN,
+       MIC_RESETTING,
        MIC_RESET_FAILED,
-       MIC_SUSPENDING,
-       MIC_SUSPENDED,
        MIC_LAST
 };