GlusterServiceSyncJobTest.java

package org.ovirt.engine.core.bll.gluster;

import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.ovirt.engine.core.utils.MockConfigRule.mockConfig;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.ArgumentMatcher;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.MockitoJUnitRunner;
import org.ovirt.engine.core.bll.utils.GlusterAuditLogUtil;
import org.ovirt.engine.core.bll.utils.GlusterUtil;
import org.ovirt.engine.core.common.businessentities.Cluster;
import org.ovirt.engine.core.common.businessentities.VDS;
import org.ovirt.engine.core.common.businessentities.VDSStatus;
import org.ovirt.engine.core.common.businessentities.gluster.GlusterClusterService;
import org.ovirt.engine.core.common.businessentities.gluster.GlusterServerService;
import org.ovirt.engine.core.common.businessentities.gluster.GlusterService;
import org.ovirt.engine.core.common.businessentities.gluster.GlusterServiceStatus;
import org.ovirt.engine.core.common.businessentities.gluster.ServiceType;
import org.ovirt.engine.core.common.config.ConfigValues;
import org.ovirt.engine.core.common.errors.EngineError;
import org.ovirt.engine.core.common.errors.VDSError;
import org.ovirt.engine.core.common.vdscommands.VDSCommandType;
import org.ovirt.engine.core.common.vdscommands.VDSReturnValue;
import org.ovirt.engine.core.common.vdscommands.gluster.GlusterServicesListVDSParameters;
import org.ovirt.engine.core.compat.Guid;
import org.ovirt.engine.core.compat.Version;
import org.ovirt.engine.core.dao.ClusterDao;
import org.ovirt.engine.core.dao.gluster.GlusterClusterServiceDao;
import org.ovirt.engine.core.dao.gluster.GlusterServerServiceDao;
import org.ovirt.engine.core.utils.MockConfigRule;

@RunWith(MockitoJUnitRunner.class)
public class GlusterServiceSyncJobTest {
    private static final Guid CLUSTER_ID = Guid.newGuid();
    private static final Guid SERVER1_ID = Guid.newGuid();
    private static final Guid SERVER2_ID = Guid.newGuid();
    private static final Guid SERVER3_ID = Guid.newGuid();
    private static final Guid SERVICE1_ID = Guid.newGuid();
    private static final Guid SERVICE2_ID = Guid.newGuid();
    private static final Guid SERVICE3_ID = Guid.newGuid();
    private static final String SERVICE1_NAME = "service1";
    private static final String SERVICE2_NAME = "service2";
    private static final String SERVICE3_NAME = "service3";
    private List<GlusterClusterService> existingClusterServices;
    private Map<String, GlusterService> serviceNameMap;

    @InjectMocks
    @Spy
    private GlusterServiceSyncJob syncJob;

    @ClassRule
    public static MockConfigRule mcr = new MockConfigRule(
            mockConfig(ConfigValues.GlusterServicesEnabled, Version.getLast(), true));

    @Mock
    private GlusterServerServiceDao serverServiceDao;

    @Mock
    private GlusterClusterServiceDao clusterServiceDao;

    @Mock
    private GlusterUtil glusterUtil;

    @Mock
    private ClusterDao clusterDao;

    @Mock
    private GlusterAuditLogUtil logUtil;

    @Before
    public void setUp() {
        createObjects();
        setupCommonMock();
    }

    private void createObjects() {
        serviceNameMap = createServiceNameMap();
        existingClusterServices = createClusterServices();
    }

    private void setupCommonMock() {
        doReturn(serviceNameMap).when(syncJob).getServiceNameMap();

        doReturn(Collections.singletonList(createCluster())).when(clusterDao).getAll();
        doReturn(createServerServices(SERVER1_ID, GlusterServiceStatus.RUNNING)).when(serverServiceDao)
                .getByServerId(SERVER1_ID);
        doReturn(createServerServices(SERVER2_ID, GlusterServiceStatus.RUNNING)).when(serverServiceDao)
                .getByServerId(SERVER2_ID);
        doReturn(createServerServices(SERVER3_ID, GlusterServiceStatus.RUNNING)).when(serverServiceDao)
                .getByServerId(SERVER3_ID);
        doReturn(existingClusterServices).when(clusterServiceDao).getByClusterId(CLUSTER_ID);
        doReturn(createServers()).when(glusterUtil).getAllUpServers(CLUSTER_ID);

        doNothing().when(syncJob).acquireLock(SERVER1_ID);
        doNothing().when(syncJob).releaseLock(SERVER1_ID);
        doNothing().when(syncJob).acquireLock(SERVER2_ID);
        doNothing().when(syncJob).releaseLock(SERVER2_ID);
        doNothing().when(syncJob).acquireLock(SERVER3_ID);
        doNothing().when(syncJob).releaseLock(SERVER3_ID);
    }

    private List<VDS> createServers() {
        List<VDS> serverList = new ArrayList<>();
        VDS server1 = createUpServer(SERVER1_ID);
        server1.setStatus(VDSStatus.Up);
        serverList.add(server1);
        VDS server2 = createUpServer(SERVER2_ID);
        server2.setStatus(VDSStatus.Up);
        serverList.add(server2);
        VDS server3 = createUpServer(SERVER3_ID);
        server3.setStatus(VDSStatus.Up);
        serverList.add(server3);

        return serverList;
    }

    private Map<String, GlusterService> createServiceNameMap() {
        Map<String, GlusterService> map = new HashMap<>();
        map.put(SERVICE1_NAME, createGlusterService(SERVICE1_ID, SERVICE1_NAME, ServiceType.GLUSTER));
        map.put(SERVICE2_NAME, createGlusterService(SERVICE2_ID, SERVICE2_NAME, ServiceType.GLUSTER_SWIFT));
        map.put(SERVICE3_NAME, createGlusterService(SERVICE3_ID, SERVICE3_NAME, ServiceType.GLUSTER_SWIFT));
        return map;
    }

    private GlusterService createGlusterService(Guid serviceId, String serviceName, ServiceType type) {
        GlusterService service = new GlusterService();
        service.setId(serviceId);
        service.setServiceName(serviceName);
        service.setServiceType(type);
        return service;
    }

    private Cluster createCluster() {
        Cluster cluster = new Cluster();
        cluster.setId(CLUSTER_ID);
        cluster.setGlusterService(true);
        cluster.setCompatibilityVersion(Version.getLast());
        return cluster;
    }

    private List<GlusterClusterService> createClusterServices() {
        List<GlusterClusterService> services = new ArrayList<>();

        services.add(createClusterService(CLUSTER_ID, ServiceType.GLUSTER_SWIFT, GlusterServiceStatus.RUNNING));
        services.add(createClusterService(CLUSTER_ID, ServiceType.GLUSTER, GlusterServiceStatus.RUNNING));

        return services;
    }

    private GlusterClusterService createClusterService(Guid clusterId,
            ServiceType serviceType,
            GlusterServiceStatus status) {
        GlusterClusterService service = new GlusterClusterService();
        service.setClusterId(clusterId);
        service.setServiceType(serviceType);
        service.setStatus(status);
        return service;
    }

    private List<GlusterServerService> createServerServices(Guid serverId, GlusterServiceStatus status) {
        List<GlusterServerService> services = new ArrayList<>();

        services.add(createServerService(serverId, SERVICE1_ID, SERVICE1_NAME, status));
        services.add(createServerService(serverId, SERVICE2_ID, SERVICE2_NAME, status));
        services.add(createServerService(serverId, SERVICE3_ID, SERVICE3_NAME, status));

        return services;
    }

    private List<GlusterServerService> createServerServicesWithMixedStatus(Guid serverId) {
        List<GlusterServerService> services = new ArrayList<>();

        services.add(createServerService(serverId, SERVICE1_ID, SERVICE1_NAME, GlusterServiceStatus.RUNNING));
        services.add(createServerService(serverId, SERVICE2_ID, SERVICE2_NAME, GlusterServiceStatus.STOPPED));
        services.add(createServerService(serverId, SERVICE3_ID, SERVICE3_NAME, GlusterServiceStatus.ERROR));

        return services;
    }

    private GlusterServerService createServerService(Guid serverId,
            Guid serviceId,
            String serviceName,
            GlusterServiceStatus status) {
        GlusterServerService service = new GlusterServerService();
        service.setId(Guid.newGuid());
        service.setServerId(serverId);
        service.setServiceId(serviceId);
        service.setStatus(status);
        service.setServiceName(serviceName);
        return service;
    }

    private VDS createUpServer(Guid serverId) {
        VDS server = new VDS();
        server.setId(serverId);
        return server;
    }

    @Test
    public void testRefreshGlusterServicesNoChanges() throws Exception {
        mockNoChanges();
        syncJob.refreshGlusterServices();
        verifyNoChanges();
    }

    @Test
    public void testRefreshGlusterServicesWithChanges() throws Exception {
        mockWithChanges();
        syncJob.refreshGlusterServices();
        verifyWithChanges();
    }

    @Test
    public void testRefreshGlusterServicesWhenVdsmVerbFails() {
        mockVdsmFailureOnServer1AndNoChangesOnOthers();
        syncJob.refreshGlusterServices();
        verifyNoChangesWithFailureOnServer1();
    }

    private void verifyCommonCalls() {
        // all clusters fetched from db
        verify(clusterDao, times(1)).getAll();

        // get all servers of the cluster
        verify(glusterUtil, times(1)).getAllUpServers(CLUSTER_ID);

        // Fetch existing services from server1
        verify(serverServiceDao, times(1)).getByServerId(SERVER1_ID);

        // Fetch existing services from server2
        verify(serverServiceDao, times(1)).getByServerId(SERVER2_ID);

        // Fetch existing services from server3
        verify(serverServiceDao, times(1)).getByServerId(SERVER3_ID);

        // Fetch services statuses from all three servers
        verify(syncJob, times(3)).runVdsCommand(eq(VDSCommandType.GlusterServicesList),
                any(GlusterServicesListVDSParameters.class));

    }

    @SuppressWarnings("unchecked")
    private void verifyNoChanges() {
        verifyCommonCalls();

        // Since there are no changes in any service status, there should be no database update
        verify(serverServiceDao, never()).updateAll(anyList());
        verify(clusterServiceDao, never()).update(any(GlusterClusterService.class));
    }

    private void verifyNoChangesWithFailureOnServer1() {
        verifyCommonCalls();

        // One update on serverServiceDao to update statuses of all services of server1
        verify(serverServiceDao, times(1)).updateAll(argThat(isCollectionOfServicesOfServer1WithStatusUnknown()));

        // two updates on clusterServiceDao to update status of each service type to MIXED
        verify(clusterServiceDao, times(2)).update(argThat(isClusterServiceWithMixedStatus()));
    }

    private void verifyWithChanges() {
        verifyCommonCalls();

        // service statuses get updated on server2 and server3
        verify(serverServiceDao, times(2)).updateAll(argThat(isListOfServersWithChangedStatus()));

        // two updates on clusterServiceDao to update status of each service type to MIXED
        verify(clusterServiceDao, times(2)).update(argThat(isClusterServiceWithMixedStatus()));
    }

    private ArgumentMatcher<GlusterClusterService> isClusterServiceWithMixedStatus() {
        return argument -> argument.getStatus() == GlusterServiceStatus.MIXED;
    }

    private ArgumentMatcher<Collection<GlusterServerService>> isCollectionOfServicesOfServer1WithStatusUnknown() {
        return serverServices -> {
            for (GlusterServerService service : serverServices) {
                // Status of all services from server1 should change to UNKNOWN.
                // Nothing else should change
                if (!(service.getServerId().equals(SERVER1_ID) && service.getStatus() == GlusterServiceStatus.UNKNOWN)) {
                    return false;
                }
            }

            return true;
        };

    }

    private ArgumentMatcher<List<GlusterServerService>> isListOfServersWithChangedStatus() {
        return serverServices -> {
            for (GlusterServerService service : serverServices) {
                // server1 has no services with changed status
                if (service.getServerId().equals(SERVER1_ID)) {
                    return false;
                }

                // on server2, only service2 and service3 have changed status.
                if (service.getServerId().equals(SERVER2_ID)) {
                    if (!(service.getServiceId().equals(SERVICE2_ID) || service.getServiceId().equals(SERVICE3_ID))) {
                        return false;
                    }
                }

                // on server3, all services have different status. so no checks required.
            }

            return true;
        };
    }

    private void mockNoChanges() {
        mockStatusForAllServicesOfServer(SERVER1_ID, GlusterServiceStatus.RUNNING);
        mockStatusForAllServicesOfServer(SERVER2_ID, GlusterServiceStatus.RUNNING);
        mockStatusForAllServicesOfServer(SERVER3_ID, GlusterServiceStatus.RUNNING);
    }

    private void mockWithChanges() {
        // no changes (all RUNNING) on server1 (all services in RUNNING status)
        mockStatusForAllServicesOfServer(SERVER1_ID, GlusterServiceStatus.RUNNING);

        // mixed status on server2
        List<GlusterServerService> server2Services = createServerServicesWithMixedStatus(SERVER2_ID);
        doReturn(createVDSReturnValue(server2Services)).when(syncJob)
                .runVdsCommand(eq(VDSCommandType.GlusterServicesList),
                        argThat(isServer(SERVER2_ID)));

        // all stopped on server3
        mockStatusForAllServicesOfServer(SERVER3_ID, GlusterServiceStatus.STOPPED);
    }

    private void mockVdsmFailureOnServer1AndNoChangesOnOthers() {
        doReturn(createVDSReturnValueForFailure()).when(syncJob)
                .runVdsCommand(eq(VDSCommandType.GlusterServicesList),
                        argThat(isServer(SERVER1_ID)));
        mockStatusForAllServicesOfServer(SERVER2_ID, GlusterServiceStatus.RUNNING);
        mockStatusForAllServicesOfServer(SERVER3_ID, GlusterServiceStatus.RUNNING);
    }

    private void mockStatusForAllServicesOfServer(Guid serverId, GlusterServiceStatus status) {
        doReturn(createVDSReturnValue(serverId, status)).when(syncJob)
                .runVdsCommand(eq(VDSCommandType.GlusterServicesList),
                        argThat(isServer(serverId)));
    }

    private ArgumentMatcher<GlusterServicesListVDSParameters> isServer(final Guid serverId) {
        return argument -> argument.getVdsId().equals(serverId);
    }

    private VDSReturnValue createVDSReturnValue(List<GlusterServerService> serverServices) {
        VDSReturnValue ret = new VDSReturnValue();
        ret.setSucceeded(true);
        ret.setReturnValue(serverServices);
        return ret;
    }

    private VDSReturnValue createVDSReturnValue(Guid serverId, GlusterServiceStatus status) {
        return createVDSReturnValue(createServerServices(serverId, status));
    }

    private VDSReturnValue createVDSReturnValueForFailure() {
        VDSReturnValue ret = new VDSReturnValue();
        ret.setSucceeded(false);
        ret.setVdsError(new VDSError(EngineError.GlusterServicesActionFailed, "VDSM Error"));
        return ret;
    }
}