geo-rep: Fixes for remote authentication and gsyncd path

- gsyncd path defaulted to /usr/libexec/glusterfs/gsyncd
- Fixes remote REST API auth issues
- Workaround to make it work with marker xlator

Signed-off-by: Aravinda VK <avishwan@redhat.com>

glustercli/cmd/georep.go

@@ -3,9 +3,11 @@ package cmd
 import (
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"net/url"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/gluster/glusterd2/pkg/restclient"
 	georepapi "github.com/gluster/glusterd2/plugins/georeplication/api"
@@ -54,12 +56,24 @@ var (
 	flagGeorepCmdForce        bool
 	flagGeorepShowAllConfig   bool
 	flagGeorepRemoteEndpoints string
+	flagRemoteUser            string
+	flagRemoteSecret          string
+	flagRemoteSecretFile      string
+	flagRemoteCacert          string
+	flagRemoteInsecure        bool
 )
 
 func init() {
 	// Geo-rep Create
 	georepCreateCmd.Flags().StringVar(&flagGeorepRemoteEndpoints, "remote-endpoints", "", "remote glusterd2 endpoints")
 	georepCreateCmd.Flags().BoolVarP(&flagGeorepCmdForce, "force", "f", false, "Force")
+	georepCreateCmd.Flags().StringVar(&flagRemoteUser, "remote-user", "glustercli", "Username for authentication")
+	georepCreateCmd.Flags().StringVar(&flagRemoteSecret, "remote-secret", "", "Password for authentication")
+	georepCreateCmd.Flags().StringVar(&flagRemoteSecretFile, "remote-secret-file", "", "Path to file which contains the secret for authentication")
+	georepCreateCmd.Flags().StringVar(&flagRemoteCacert, "remote-cacert", "", "Path to CA certificate")
+	georepCreateCmd.Flags().BoolVar(&flagRemoteInsecure, "remote-insecure", false,
+		"Skip remote server certificate validation")
+
 	georepCmd.AddCommand(georepCreateCmd)
 
 	// Geo-rep Start
@@ -335,7 +349,6 @@ var georepDeleteCmd = &cobra.Command{
 }
 
 func getRemoteClient(host string) (string, *restclient.Client, error) {
-	// TODO: Handle Remote Cluster Authentication and certificates and URL scheme
 	clienturl := flagGeorepRemoteEndpoints
 
 	if flagGeorepRemoteEndpoints != "" {
@@ -346,8 +359,49 @@ func getRemoteClient(host string) (string, *restclient.Client, error) {
 	} else {
 		clienturl = fmt.Sprintf("%s://%s:%d", geoRepHTTPScheme, host, geoRepGlusterdPort)
 	}
-	client, err := restclient.New(clienturl, "", "", "", true)
-	return clienturl, client, err
+
+	remoteSecret := ""
+	// Secret is taken in following order of precedence (highest to lowest):
+	// --remote-secret
+	// --remote-secret-file
+	// GD2_REMOTE_AUTH_SECRET (environment variable)
+	// Secret set for Master cluster itself
+	// Default secret
+
+	// Remote Cluster secret --remote-secret
+	if flagRemoteSecret != "" {
+		remoteSecret = flagRemoteSecret
+	}
+
+	// Remote Cluster's secret file --remote-secret-file
+	if flagRemoteSecretFile != "" && remoteSecret == "" {
+		data, err := ioutil.ReadFile(flagRemoteSecretFile)
+		if err != nil {
+			failure(fmt.Sprintf("failed to read remote secret file %s", flagRemoteSecretFile),
+				err, 1)
+		}
+		remoteSecret = string(data)
+	}
+
+	// GD2_REMOTE_AUTH_SECRET
+	if remoteSecret == "" {
+		remoteSecret = os.Getenv("GD2_REMOTE_AUTH_SECRET")
+	}
+
+	// Below option of local cluster is used because --remote-* options
+	// are not specified and Remote volume may exists in same cluster where
+	// Master Volume exists
+	if remoteSecret == "" {
+		remoteSecret = GlobalFlag.Secret
+	}
+
+	client, err := restclient.New(clienturl, flagRemoteUser, remoteSecret, flagRemoteCacert, flagRemoteInsecure)
+	if err != nil {
+		failure("failed to setup remote client", err, 1)
+	}
+	client.SetTimeout(time.Duration(GlobalFlag.Timeout) * time.Second)
+
+	return clienturl, client, nil
 }
 
 func getVolIDs(pargs []string) (string, string, error) {

glustercli/cmd/root.go

@@ -95,7 +95,7 @@ func (gOpt *GlustercliOption) AddPersistentFlag(flagSet *pflag.FlagSet) {
 	// SSL/TLS options
 	flagSet.StringVarP(&gOpt.Cacert, "cacert", "", "", "Path to CA certificate")
 	flagSet.BoolVarP(&gOpt.Insecure, "insecure", "", false,
-		"Accepts any certificate presented by the server and any host name in that certificate.")
+		"Skip server certificate validation")
 }
 
 //Init will initialize logging, secret and rest client

plugins/georeplication/gsyncd.go

@@ -3,17 +3,35 @@ package georeplication
 import (
 	"fmt"
 	"path"
+	"strings"
 
 	"github.com/gluster/glusterd2/glusterd2/gdctx"
+	"github.com/gluster/glusterd2/pkg/utils"
 	georepapi "github.com/gluster/glusterd2/plugins/georeplication/api"
 
 	config "github.com/spf13/viper"
 )
 
-const (
-	gsyncdCommand = "/usr/local/libexec/glusterfs/gsyncd"
+var (
+	defaultGsyncdCommand = "/usr/libexec/glusterfs/gsyncd"
+	gsyncdCommand        = ""
 )
 
+func getGsyncdCommand() string {
+	if gsyncdCommand != "" {
+		return gsyncdCommand
+	}
+
+	out, err := utils.ExecuteCommandOutput("glusterfsd", "--print-libexecdir")
+	if err != nil {
+		gsyncdCommand = defaultGsyncdCommand
+		return gsyncdCommand
+	}
+
+	gsyncdCommand = path.Join(strings.TrimRight(string(out), "\n"), "gsyncd")
+	return gsyncdCommand
+}
+
 // Gsyncd type represents information about Gsyncd process
 type Gsyncd struct {
 	// Externally consumable using methods of Gsyncd interface
@@ -85,7 +103,7 @@ func (g *Gsyncd) PidFile() string {
 
 // newGsyncd returns a new instance of Gsyncd monitor type which implements the Daemon interface
 func newGsyncd(sessioninfo georepapi.GeorepSession) (*Gsyncd, error) {
-	return &Gsyncd{binarypath: gsyncdCommand, sessioninfo: sessioninfo}, nil
+	return &Gsyncd{binarypath: getGsyncdCommand(), sessioninfo: sessioninfo}, nil
 }
 
 // ID returns the unique identifier of the gsyncd.

plugins/georeplication/rest.go

@@ -174,6 +174,12 @@ func georepCreateHandler(w http.ResponseWriter, r *http.Request) {
 	// Workaround till {{ volume.id }} added to the marker options table
 	vol.Options["marker.volume-uuid"] = vol.ID.String()
 
+	// Workaround till {{ workdir }} added to the marker options table
+	vol.Options["marker.timestamp-file"] = path.Join(
+		config.GetString("localstatedir"),
+		"{{ volume.name }}.marker.tstamp",
+	)
+
 	//save volume information for transaction failure scenario
 	if err := txn.Ctx.Set("oldvolinfo", oldvolinfo); err != nil {
 		logger.WithError(err).Error("failed to set oldvolinfo in transaction context")
@@ -610,7 +616,7 @@ func checkConfig(name string, value string) error {
 	if value != "" {
 		args = append(args, "--value", value)
 	}
-	return utils.ExecuteCommandRun(gsyncdCommand, args...)
+	return utils.ExecuteCommandRun(getGsyncdCommand(), args...)
 }
 
 func georepConfigGetHandler(w http.ResponseWriter, r *http.Request) {
@@ -646,7 +652,7 @@ func georepConfigGetHandler(w http.ResponseWriter, r *http.Request) {
 		"--show-defaults",
 		"--json",
 	}
-	out, err := utils.ExecuteCommandOutput(gsyncdCommand, args...)
+	out, err := utils.ExecuteCommandOutput(getGsyncdCommand(), args...)
 	if err != nil {
 		logger.WithError(err).WithFields(log.Fields{
 			"mastervolid": masterid,

plugins/georeplication/transactions.go

@@ -166,7 +166,7 @@ func txnGeorepStatus(c transaction.TxnCtx) error {
 		}
 		args := gsyncd.statusArgs(w.Path)
 
-		out, err := utils.ExecuteCommandOutput(gsyncdCommand, args...)
+		out, err := utils.ExecuteCommandOutput(getGsyncdCommand(), args...)
 		if err != nil {
 			return err
 		}
@@ -415,7 +415,7 @@ func txnSSHKeysPush(c transaction.TxnCtx) error {
 		return err
 	}
 
-	sshCmdGsyncdPrefix := "command=\"" + gsyncdCommand + "\"  "
+	sshCmdGsyncdPrefix := "command=\"" + getGsyncdCommand() + "\"  "
 	sshCmdTarPrefix := "command=\"tar ${SSH_ORIGINAL_COMMAND#* }\"  "
 	authorizedKeysFile := "/root/.ssh/authorized_keys"