Merge branch 'master' into staging-client

ConsoleClient/main.go

@@ -161,22 +161,22 @@ func main() {
 	// Handle required config file parameter
 
 	// EmitDiagnosticNotices is set by LoadConfig; force to true
-	// an emit diagnostics when LoadConfig-related errors occur.
+	// and emit diagnostics when LoadConfig-related errors occur.
 
 	if configFilename == "" {
-		psiphon.SetEmitDiagnosticNotices(true)
+		psiphon.SetEmitDiagnosticNotices(true, false)
 		psiphon.NoticeError("configuration file is required")
 		os.Exit(1)
 	}
 	configFileContents, err := ioutil.ReadFile(configFilename)
 	if err != nil {
-		psiphon.SetEmitDiagnosticNotices(true)
+		psiphon.SetEmitDiagnosticNotices(true, false)
 		psiphon.NoticeError("error loading configuration file: %s", err)
 		os.Exit(1)
 	}
 	config, err := psiphon.LoadConfig(configFileContents)
 	if err != nil {
-		psiphon.SetEmitDiagnosticNotices(true)
+		psiphon.SetEmitDiagnosticNotices(true, false)
 		psiphon.NoticeError("error processing configuration file: %s", err)
 		os.Exit(1)
 	}
@@ -191,7 +191,7 @@ func main() {
 		tunDeviceFile, err := configurePacketTunnel(
 			config, tunDevice, tunBindInterface, tunPrimaryDNS, tunSecondaryDNS)
 		if err != nil {
-			psiphon.SetEmitDiagnosticNotices(true)
+			psiphon.SetEmitDiagnosticNotices(true, false)
 			psiphon.NoticeError("error configuring packet tunnel: %s", err)
 			os.Exit(1)
 		}
@@ -202,7 +202,7 @@ func main() {
 
 	err = config.Commit()
 	if err != nil {
-		psiphon.SetEmitDiagnosticNotices(true)
+		psiphon.SetEmitDiagnosticNotices(true, false)
 		psiphon.NoticeError("error loading configuration file: %s", err)
 		os.Exit(1)
 	}

MobileLibrary/Android/PsiphonTunnel/PsiphonTunnel.java

@@ -22,17 +22,16 @@
 import android.annotation.TargetApi;
 import android.content.Context;
 import android.net.ConnectivityManager;
-import android.net.wifi.WifiManager;
-import android.net.wifi.WifiInfo;
 import android.net.LinkProperties;
 import android.net.NetworkInfo;
 import android.net.VpnService;
+import android.net.wifi.WifiInfo;
+import android.net.wifi.WifiManager;
 import android.os.Build;
 import android.os.ParcelFileDescriptor;
 import android.telephony.TelephonyManager;
 import android.util.Base64;
 
-import org.apache.http.conn.util.InetAddressUtils;
 import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
@@ -44,6 +43,7 @@
 import java.io.PrintStream;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
+import java.net.Inet4Address;
 import java.net.InetAddress;
 import java.net.NetworkInterface;
 import java.net.SocketException;
@@ -190,6 +190,37 @@ public synchronized void restartPsiphon() throws Exception {
         startPsiphon("");
     }
 
+    // Creates a temporary dummy VPN interface in order to prevent traffic leaking while performing
+    // complete VPN and tunnel restart, for example, caused by host app settings change.
+    // Note: same deadlock note as stop().
+    @TargetApi(Build.VERSION_CODES.ICE_CREAM_SANDWICH)
+    public synchronized void seamlessVpnRestart(VpnService.Builder vpnServiceBuilder) throws Exception {
+        // Perform seamless VPN interface swap Psiphon VPN -> dummy VPN
+        //
+        // From https://developer.android.com/reference/android/net/VpnService.Builder.html#establish()
+        // "However, it is rare but not impossible to have two interfaces while performing a seamless handover.
+        // In this case, the old interface will be deactivated when the new one is created successfully. Both
+        // file descriptors are valid but now outgoing packets will be routed to the new interface. Therefore,
+        // after draining the old file descriptor, the application MUST close it and start using the new file
+        // descriptor."
+        ParcelFileDescriptor dummyVpnFd = startDummyVpn(vpnServiceBuilder);
+        try {
+            // Clean up and restart Psiphon VPN interface, which will also do the swap dummy VPN -> Psiphon VPN
+            stopVpn();
+            startVpn();
+        } finally {
+            // Close dummy VPN file descriptor as per documentation.
+            if (dummyVpnFd != null) {
+                try {
+                    dummyVpnFd.close();
+                } catch (IOException e) {
+                }
+            }
+        }
+        // Restart the tunnel.
+        restartPsiphon();
+    }
+
     public void setClientPlatformAffixes(String prefix, String suffix) {
         mClientPlatformPrefix.set(prefix);
         mClientPlatformSuffix.set(suffix);
@@ -279,6 +310,37 @@ private boolean startVpn() throws Exception {
         return true;
     }
 
+    @TargetApi(Build.VERSION_CODES.ICE_CREAM_SANDWICH)
+    private ParcelFileDescriptor startDummyVpn(VpnService.Builder vpnServiceBuilder) throws Exception {
+        PrivateAddress privateAddress = selectPrivateAddress();
+
+        Locale previousLocale = Locale.getDefault();
+
+        final String errorMessage = "startDummyVpn failed";
+        final ParcelFileDescriptor tunFd;
+        try {
+            // Workaround for https://code.google.com/p/android/issues/detail?id=61096
+            Locale.setDefault(new Locale("en"));
+            tunFd = vpnServiceBuilder
+                            .setSession(mHostService.getAppName())
+                            .addAddress(mPrivateAddress.mIpAddress, mPrivateAddress.mPrefixLength)
+                            .addRoute("0.0.0.0", 0)
+                            .addRoute(mPrivateAddress.mSubnet, mPrivateAddress.mPrefixLength)
+                            .establish();
+        } catch(IllegalArgumentException e) {
+            throw new Exception(errorMessage, e);
+        } catch(IllegalStateException e) {
+            throw new Exception(errorMessage, e);
+        } catch(SecurityException e) {
+            throw new Exception(errorMessage, e);
+        } finally {
+            // Restore the original locale.
+            Locale.setDefault(previousLocale);
+        }
+
+        return tunFd;
+    }
+
     private boolean isVpnMode() {
         return mVpnMode.get();
     }
@@ -902,8 +964,8 @@ private static PrivateAddress selectPrivateAddress() throws Exception {
 
         for (NetworkInterface netInterface : netInterfaces) {
             for (InetAddress inetAddress : Collections.list(netInterface.getInetAddresses())) {
-                String ipAddress = inetAddress.getHostAddress();
-                if (InetAddressUtils.isIPv4Address(ipAddress)) {
+                if (inetAddress instanceof Inet4Address) {
+                    String ipAddress = inetAddress.getHostAddress();
                     if (ipAddress.startsWith("10.")) {
                         candidates.remove("10");
                     }

MobileLibrary/Android/make.bash

@@ -55,7 +55,8 @@ echo " Gomobile version: ${GOMOBILEVERSION}"
 echo " Dependencies: ${DEPENDENCIES}"
 echo ""
 
-gomobile bind -v -x -target=android/arm,android/arm64 -tags="${BUILD_TAGS}" -ldflags="$LDFLAGS" github.com/Psiphon-Labs/psiphon-tunnel-core/MobileLibrary/psi
+# Note: android/386 is x86 and android/amd64 is x86_64
+gomobile bind -v -x -target=android/arm,android/arm64,android/386,android/amd64 -tags="${BUILD_TAGS}" -ldflags="$LDFLAGS" github.com/Psiphon-Labs/psiphon-tunnel-core/MobileLibrary/psi
 if [ $? != 0 ]; then
   echo "..'gomobile bind' failed, exiting"
   exit $?
@@ -66,6 +67,8 @@ unzip -o psi.aar -d build-tmp/psi
 yes | cp -f PsiphonTunnel/AndroidManifest.xml build-tmp/psi/AndroidManifest.xml
 yes | cp -f PsiphonTunnel/libs/armeabi-v7a/libtun2socks.so build-tmp/psi/jni/armeabi-v7a/libtun2socks.so
 yes | cp -f PsiphonTunnel/libs/arm64-v8a/libtun2socks.so build-tmp/psi/jni/arm64-v8a/libtun2socks.so
+yes | cp -f PsiphonTunnel/libs/x86/libtun2socks.so build-tmp/psi/jni/x86/libtun2socks.so
+yes | cp -f PsiphonTunnel/libs/x86_64/libtun2socks.so build-tmp/psi/jni/x86_64/libtun2socks.so
 
 javac -d build-tmp -bootclasspath $ANDROID_HOME/platforms/android-23/android.jar -source 1.7 -target 1.7 -classpath build-tmp/psi/classes.jar:$ANDROID_HOME/platforms/android-23/optional/org.apache.http.legacy.jar PsiphonTunnel/PsiphonTunnel.java
 if [ $? != 0 ]; then

psiphon/LookupIP.go

@@ -62,12 +62,20 @@ func LookupIP(ctx context.Context, host string, config *DialConfig) ([]net.IP, e
 			return ips, err
 		}
 
-		NoticeAlert("retry resolve host %s: %s", host, err)
+		if GetEmitNetworkParameters() {
+			NoticeAlert("retry resolve host %s: %s", host, err)
+		}
 
 		return bindLookupIP(ctx, host, dnsServer, config)
 	}
 
 	addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
+
+	// Remove domain names from "net" error messages.
+	if err != nil && !GetEmitNetworkParameters() {
+		err = RedactNetError(err)
+	}
+
 	if err != nil {
 		return nil, common.ContextError(err)
 	}
@@ -116,7 +124,7 @@ func bindLookupIP(
 		copy(ipv6[:], ipAddr.To16())
 		domain = syscall.AF_INET6
 	} else {
-		return nil, common.ContextError(fmt.Errorf("invalid IP address for dns server: %s", ipAddr.String()))
+		return nil, common.ContextError(errors.New("invalid IP address for DNS server"))
 	}
 
 	socketFd, err := syscall.Socket(domain, syscall.SOCK_DGRAM, 0)
@@ -127,7 +135,7 @@ func bindLookupIP(
 	_, err = config.DeviceBinder.BindToDevice(socketFd)
 	if err != nil {
 		syscall.Close(socketFd)
-		return nil, common.ContextError(fmt.Errorf("BindToDevice failed: %s", err))
+		return nil, common.ContextError(fmt.Errorf("BindToDevice failed with %s", err))
 	}
 
 	// Connect socket to the server's IP address

psiphon/LookupIP_nobind.go

@@ -37,6 +37,12 @@ func LookupIP(ctx context.Context, host string, config *DialConfig) ([]net.IP, e
 	}
 
 	addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
+
+	// Remove domain names from "net" error messages.
+	if err != nil && !GetEmitNetworkParameters() {
+		err = RedactNetError(err)
+	}
+
 	if err != nil {
 		return nil, common.ContextError(err)
 	}

psiphon/TCPConn.go

@@ -84,7 +84,9 @@ func DialTCP(
 	if config.FragmentorConfig.MayFragment() {
 		conn = fragmentor.NewConn(
 			config.FragmentorConfig,
-			func(message string) { NoticeInfo(message) },
+			func(message string) {
+				NoticeFragmentor(config.DiagnosticID, message)
+			},
 			conn)
 	}
 

psiphon/TCPConn_bind.go

@@ -117,7 +117,7 @@ func tcpDial(ctx context.Context, addr string, config *DialConfig) (net.Conn, er
 			copy(ipv6[:], ipAddr.To16())
 			domain = syscall.AF_INET6
 		} else {
-			lastErr = common.ContextError(fmt.Errorf("invalid IP address: %s", ipAddr.String()))
+			lastErr = common.ContextError(errors.New("invalid IP address"))
 			continue
 		}
 		if domain == syscall.AF_INET {
@@ -142,7 +142,7 @@ func tcpDial(ctx context.Context, addr string, config *DialConfig) (net.Conn, er
 			_, err = config.DeviceBinder.BindToDevice(socketFD)
 			if err != nil {
 				syscall.Close(socketFD)
-				lastErr = common.ContextError(fmt.Errorf("BindToDevice failed: %s", err))
+				lastErr = common.ContextError(fmt.Errorf("BindToDevice failed with %s", err))
 				continue
 			}
 		}

psiphon/TCPConn_nobind.go

@@ -40,6 +40,11 @@ func tcpDial(ctx context.Context, addr string, config *DialConfig) (net.Conn, er
 
 	conn, err := dialer.DialContext(ctx, "tcp", addr)
 
+	// Remove domain names from "net" error messages.
+	if err != nil && !GetEmitNetworkParameters() {
+		err = RedactNetError(err)
+	}
+
 	if err != nil {
 		return nil, common.ContextError(err)
 	}

psiphon/common/fragmentor/fragmentor.go

@@ -259,14 +259,7 @@ func (c *Conn) Write(buffer []byte) (int, error) {
 	var notice bytes.Buffer
 
 	if emitNotice {
-		remoteAddrStr := "(nil)"
-		remoteAddr := c.Conn.RemoteAddr()
-		if remoteAddr != nil {
-			remoteAddrStr = remoteAddr.String()
-		}
-		fmt.Fprintf(&notice,
-			"fragment %s %d bytes:",
-			remoteAddrStr, len(buffer))
+		fmt.Fprintf(&notice, "fragment %d bytes:", len(buffer))
 	}
 
 	for iterations := 0; len(buffer) > 0; iterations += 1 {

psiphon/common/protocol/serverEntry.go

@@ -155,6 +155,18 @@ func (fields ServerEntryFields) SetTag(tag string) {
 	fields["isLocalDerivedTag"] = true
 }
 
+func (fields ServerEntryFields) GetDiagnosticID() string {
+	tag, ok := fields["tag"]
+	if !ok {
+		return ""
+	}
+	tagStr, ok := tag.(string)
+	if !ok {
+		return ""
+	}
+	return TagToDiagnosticID(tagStr)
+}
+
 func (fields ServerEntryFields) GetIPAddress() string {
 	ipAddress, ok := fields["ipAddress"]
 	if !ok {
@@ -279,6 +291,15 @@ func (fields ServerEntryFields) AddSignature(publicKey, privateKey string) error
 
 	delete(copyFields, "signature")
 
+	// Limitation: since the verifyier must remarshal its server entry before
+	// verifying, the JSON produced there must be a byte-for-byte match to the
+	// JSON signed here. The precise output of the JSON encoder that is used,
+	// "encoding/json", with default formatting, as of Go 1.11.5, is therefore
+	// part of the signature protocol.
+	//
+	// TODO: use a stadard, canonical encoding, such as JCS:
+	// https://tools.ietf.org/id/draft-rundgren-json-canonicalization-scheme-05.html
+
 	marshaledFields, err := json.Marshal(copyFields)
 	if err != nil {
 		return common.ContextError(err)
@@ -507,6 +528,10 @@ func (serverEntry *ServerEntry) HasSignature() bool {
 	return serverEntry.Signature != ""
 }
 
+func (serverEntry *ServerEntry) GetDiagnosticID() string {
+	return TagToDiagnosticID(serverEntry.Tag)
+}
+
 // GenerateServerEntryTag creates a server entry tag value that is
 // cryptographically derived from the IP address and web server secret in a
 // way that is difficult to reverse the IP address value from the tag or
@@ -521,6 +546,18 @@ func GenerateServerEntryTag(ipAddress, webServerSecret string) string {
 	return base64.StdEncoding.EncodeToString(h.Sum(nil))
 }
 
+// TagToDiagnosticID returns a prefix of the server entry tag that should be
+// sufficient to uniquely identify servers in diagnostics, while also being
+// more human readable than emitting the full tag. The tag is used as the base
+// of the diagnostic ID as it doesn't leak the server IP address in diagnostic
+// output.
+func TagToDiagnosticID(tag string) string {
+	if len(tag) < 8 {
+		return "<unknown>"
+	}
+	return tag[:8]
+}
+
 // EncodeServerEntry returns a string containing the encoding of
 // a ServerEntry following Psiphon conventions.
 func EncodeServerEntry(serverEntry *ServerEntry) (string, error) {

psiphon/config.go

@@ -447,10 +447,16 @@ type Config struct {
 
 	// EmitDiagnosticNotices indicates whether to output notices containing
 	// detailed information about the Psiphon session. As these notices may
-	// contain sensitive network information, they should not be insecurely
-	// distributed or displayed to users. Default is off.
+	// contain sensitive information, they should not be insecurely distributed
+	// or displayed to users. Default is off.
 	EmitDiagnosticNotices bool
 
+	// EmitDiagnosticNetworkParameters indicates whether to include network
+	// parameters in diagnostic notices. As these parameters are sensitive
+	// circumvention network information, they should not be insecurely
+	// distributed or displayed to users. Default is off.
+	EmitDiagnosticNetworkParameters bool
+
 	// RateLimits specify throttling configuration for the tunnel.
 	RateLimits common.RateLimits
 
@@ -596,10 +602,11 @@ func (config *Config) IsCommitted() bool {
 // not be reflected in internal data structures.
 func (config *Config) Commit() error {
 
-	// Do SetEmitDiagnosticNotices first, to ensure config file errors are emitted.
-
+	// Do SetEmitDiagnosticNotices first, to ensure config file errors are
+	// emitted.
 	if config.EmitDiagnosticNotices {
-		SetEmitDiagnosticNotices(true)
+		SetEmitDiagnosticNotices(
+			true, config.EmitDiagnosticNetworkParameters)
 	}
 
 	// Promote legacy fields.
@@ -1265,7 +1272,7 @@ func (n *loggingNetworkIDGetter) GetNetworkID() string {
 	}
 	if len(logNetworkID)+1 < len(networkID) {
 		// Indicate when additional network info was present after the first "-".
-		logNetworkID += "+<network info>"
+		logNetworkID += "+[redacted]"
 	}
 	NoticeNetworkID(logNetworkID)