[209704] [api] Ability to override default encoding conversion needed.

rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/internal/services/dstore/files/DStoreFileService.java

@@ -27,6 +27,7 @@
  * David McKnight   (IBM)        - [209552] API changes to use multiple and getting rid of deprecated
  * David McKnight   (IBM)        - [210109] store constants in IFileService rather than IFileServiceConstants
  * David McKnight   (IBM)        - [210812] for text transfer, need to honour the preference (instead of straight binary)
+ * David McKnight   (IBM)        - [209704] [api] Ability to override default encoding conversion needed.
  ********************************************************************************/
 
 package org.eclipse.rse.internal.services.dstore.files;
@@ -44,8 +45,15 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Vector;
 
+import org.eclipse.core.runtime.CoreException;
+import org.eclipse.core.runtime.IConfigurationElement;
+import org.eclipse.core.runtime.IExtension;
+import org.eclipse.core.runtime.IExtensionPoint;
+import org.eclipse.core.runtime.IExtensionRegistry;
 import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.core.runtime.Platform;
 import org.eclipse.dstore.core.model.DE;
 import org.eclipse.dstore.core.model.DataElement;
 import org.eclipse.dstore.core.model.DataStore;
@@ -69,7 +77,9 @@ import org.eclipse.rse.services.clientserver.messages.SystemMessageException;
 import org.eclipse.rse.services.dstore.AbstractDStoreService;
 import org.eclipse.rse.services.dstore.util.DownloadListener;
 import org.eclipse.rse.services.dstore.util.FileSystemMessageUtil;
+import org.eclipse.rse.services.files.DefaultFileServiceCodePageConverter;
 import org.eclipse.rse.services.files.IFileService;
+import org.eclipse.rse.services.files.IFileServiceCodePageConverter;
 import org.eclipse.rse.services.files.IHostFile;
 import org.eclipse.rse.services.files.RemoteFileSecurityException;
 
@@ -84,6 +94,8 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer
 	private int _bufferDownloadSize = IUniversalDataStoreConstants.BUFFER_SIZE;
 	protected ISystemFileTypes _fileTypeRegistry;
 	private String remoteEncoding;
+	private IFileServiceCodePageConverter _defaultCodePageConverter;
+	protected Vector _codePageConverters;
 	
 	protected boolean unixStyle = false;
 	
@@ -115,6 +127,8 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer
 		_uploadLogElement = null;
 	}
 	
+
+	
 	public String getName()
 	{
 		return ServiceResources.DStore_File_Service_Label;
@@ -207,6 +221,7 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer
 	}
 	
 	
+
 	protected DataElement getAttributes(String fileNameFilter, boolean showHidden)
 	{
 		DataStore ds = getDataStore();
@@ -420,6 +435,60 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer
 		return true;
 	}
 
+	protected IFileServiceCodePageConverter getDefaultCodePageConverter()
+	{
+		if (_defaultCodePageConverter == null){
+			_defaultCodePageConverter = new DefaultFileServiceCodePageConverter();
+		}
+		return _defaultCodePageConverter;
+	}
+	
+	/**
+	 * Retrieves the first codepage converter provided via the codePageConverter extension point for the specified 
+	 * encoding
+	 * @param serverEncoding	The server encoding for which to retrieve a code page converter 
+	 * @return	A code page converter for the specified encoding, or null if no converter was found for that encoding.
+	 */
+	protected IFileServiceCodePageConverter getCodePageConverter(String serverEncoding) {
+		if (_codePageConverters == null) {
+			// retrieve all extension points
+        	IExtensionRegistry registry = Platform.getExtensionRegistry();
+			IExtensionPoint ep = registry.getExtensionPoint("org.eclipse.rse.services", "codePageConverter"); //$NON-NLS-1$
+			if (ep != null){
+			IExtension[] extensions = ep.getExtensions();
+			_codePageConverters = new Vector();
+			for (int i = 0; i < extensions.length; i++) {
+				IExtension extension = extensions[i];
+				IConfigurationElement[] configElements = extension.getConfigurationElements();
+				for (int j = 0; j < configElements.length; j++) {
+					IConfigurationElement element = configElements[j];
+					if (element.getName().equalsIgnoreCase("codePageConverter")) {								
+						try {
+							Object codePageConverter = element.createExecutableExtension("class");
+							if (codePageConverter!=null && codePageConverter instanceof IFileServiceCodePageConverter)
+								// only save extension point which implement the correct interface
+								_codePageConverters.add(codePageConverter);
+						} catch (CoreException e) {
+							//shouldn't get here....
+						}
+					}
+				}
+			}
+			}
+        }
+		if (_codePageConverters != null)
+		{
+		//scan through the available converters and return the first valid one for the specified encoding for this 
+		// subsystem implementation
+		for (int i=0; i<_codePageConverters.size(); i++) {
+			IFileServiceCodePageConverter codePageConverter = (IFileServiceCodePageConverter)_codePageConverters.elementAt(i); 
+			if (codePageConverter.isServerEncodingSupported(serverEncoding, this))
+				return codePageConverter;
+		}
+		}
+		return null;
+	}
+	
 	public boolean upload(File file, String remoteParent, String remoteFile, boolean isBinary,
 			String srcEncoding, String hostEncoding, IProgressMonitor monitor)
 	{
@@ -480,6 +549,11 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer
 			
 			int localLineSepLength = localLineSep.length();
 			
+			IFileServiceCodePageConverter codePageConverter = getCodePageConverter(hostEncoding);
+			if (codePageConverter == null) {
+				codePageConverter = getDefaultCodePageConverter();
+			}
+
 			// upload bytes while available
 			while (available > 0 && !isCancelled)
 			{
@@ -529,7 +603,8 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer
 						}
 					}
 
-					convBytes = tempStr.getBytes(hostEncoding);
+					
+					convBytes = codePageConverter.convertClientStringToRemoteBytes(tempStr, hostEncoding, this);
 				
 					if (first)
 					{ // send first set of bytes
@@ -675,7 +750,8 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer
 		DataStore ds = getDataStore();
 		DataElement universaltemp = getMinerElement();
 		
-		int mode = isBinary ? IUniversalDataStoreConstants.BINARY_MODE : IUniversalDataStoreConstants.TEXT_MODE; 
+		//int mode = isBinary ? IUniversalDataStoreConstants.BINARY_MODE : IUniversalDataStoreConstants.TEXT_MODE; 
+		int mode = IUniversalDataStoreConstants.BINARY_MODE;
 		
 		if (!makeSureLocalExists(localFile))
 		{
@@ -771,6 +847,17 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer
 
 			if (resultChild.getType().equals(IUniversalDataStoreConstants.DOWNLOAD_RESULT_SUCCESS_TYPE))
 			{								
+				if (!isBinary){ // do standard conversion if this is text!	
+					String localEncoding = System.getProperty("file.encoding");
+					IFileServiceCodePageConverter codePageConverter = getCodePageConverter(encoding);
+					if (codePageConverter == null) {
+						codePageConverter = getDefaultCodePageConverter();
+					}
+					codePageConverter.convertFileFromRemoteEncoding(localFile, encoding, localEncoding, this);
+				}
+					
+				
+				
 				return true;
 			}
 			else if (resultChild.getType().equals(IUniversalDataStoreConstants.DOWNLOAD_RESULT_FILE_NOT_FOUND_EXCEPTION))
@@ -880,7 +967,7 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer
 		// kick off all downloads
 		for (int i = 0; i < des.length && result == true; i++)
 		{
-			int mode = isBinaries[i] ? IUniversalDataStoreConstants.BINARY_MODE : IUniversalDataStoreConstants.TEXT_MODE; 
+			int mode = IUniversalDataStoreConstants.BINARY_MODE;
 			DataElement de = des[i];
 			String remotePath = paths[i];
 			
@@ -990,6 +1077,16 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer
 
 				if (resultChild.getType().equals(IUniversalDataStoreConstants.DOWNLOAD_RESULT_SUCCESS_TYPE))
 				{
+					// do standard conversion if this is text!
+					if (!isBinaries[i]){ // do standard conversion if this is text!	
+						String localEncoding = System.getProperty("file.encoding");
+						IFileServiceCodePageConverter codePageConverter = getCodePageConverter(hostEncodings[i]);
+						if (codePageConverter == null) {
+							codePageConverter = getDefaultCodePageConverter();
+						}
+						codePageConverter.convertFileFromRemoteEncoding(localFile, hostEncodings[i], localEncoding, this);
+					}
+					
 					result = true;
 				}
 				else if (resultChild.getType().equals(IUniversalDataStoreConstants.DOWNLOAD_RESULT_FILE_NOT_FOUND_EXCEPTION))

rse/plugins/org.eclipse.rse.services/src/org/eclipse/rse/services/files/AbstractFileService.java

@@ -18,6 +18,7 @@
  * David McKnight   (IBM)        - [209552] API changes to use multiple and getting rid of deprecated
  * David McKnight   (IBM)        - [210109] store constants in IFileService rather than IFileServiceConstants
  * Martin Oberhuber (Wind River) - [210109] no need to declare IFileService constants in AbstractFileService
+ * David McKnight   (IBM)        - [209704] [api] Ability to override default encoding conversion needed.
  ********************************************************************************/
 
 package org.eclipse.rse.services.files;
@@ -27,8 +28,15 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Vector;
 
+import org.eclipse.core.runtime.CoreException;
+import org.eclipse.core.runtime.IConfigurationElement;
+import org.eclipse.core.runtime.IExtension;
+import org.eclipse.core.runtime.IExtensionPoint;
+import org.eclipse.core.runtime.IExtensionRegistry;
 import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.core.runtime.Platform;
 
 import org.eclipse.rse.services.clientserver.messages.SystemMessage;
 import org.eclipse.rse.services.clientserver.messages.SystemMessageException;
@@ -36,6 +44,8 @@ import org.eclipse.rse.services.clientserver.messages.SystemMessageException;
 
 public abstract class AbstractFileService implements IFileService
 {
+	protected Vector _codePageConverters;
+	private IFileServiceCodePageConverter _defaultCodePageConverter;
 	
 	protected abstract IHostFile[] internalFetch(String parentPath, String fileFilter, int fileType, IProgressMonitor monitor) throws SystemMessageException;
 	
@@ -209,4 +219,58 @@ public abstract class AbstractFileService implements IFileService
 	public OutputStream getOutputStream(String remoteParent, String remoteFile, boolean isBinary, IProgressMonitor monitor) throws SystemMessageException {
 		return null;
 	}
+	
+	protected IFileServiceCodePageConverter getDefaultCodePageConverter()
+	{
+		if (_defaultCodePageConverter == null){
+			_defaultCodePageConverter = new DefaultFileServiceCodePageConverter();
+		}
+		return _defaultCodePageConverter;
+	}
+	
+	/**
+	 * Retrieves the first codepage converter provided via the codePageConverter extension point for the specified 
+	 * encoding
+	 * @param serverEncoding	The server encoding for which to retrieve a code page converter 
+	 * @return	A code page converter for the specified encoding, or null if no converter was found for that encoding.
+	 */
+	protected IFileServiceCodePageConverter getCodePageConverter(String serverEncoding) {
+		if (_codePageConverters == null) {
+			// retrieve all extension points
+        	IExtensionRegistry registry = Platform.getExtensionRegistry();
+			IExtensionPoint ep = registry.getExtensionPoint("org.eclipse.rse.services", "codePageConverter"); //$NON-NLS-1$
+			if (ep != null){
+			IExtension[] extensions = ep.getExtensions();
+			_codePageConverters = new Vector();
+			for (int i = 0; i < extensions.length; i++) {
+				IExtension extension = extensions[i];
+				IConfigurationElement[] configElements = extension.getConfigurationElements();
+				for (int j = 0; j < configElements.length; j++) {
+					IConfigurationElement element = configElements[j];
+					if (element.getName().equalsIgnoreCase("codePageConverter")) {								
+						try {
+							Object codePageConverter = element.createExecutableExtension("class");
+							if (codePageConverter!=null && codePageConverter instanceof IFileServiceCodePageConverter)
+								// only save extension point which implement the correct interface
+								_codePageConverters.add(codePageConverter);
+						} catch (CoreException e) {
+							//shouldn't get here....
+						}
+					}
+				}
+			}
+			}
+        }
+		if (_codePageConverters != null)
+		{
+		//scan through the available converters and return the first valid one for the specified encoding for this 
+		// subsystem implementation
+		for (int i=0; i<_codePageConverters.size(); i++) {
+			IFileServiceCodePageConverter codePageConverter = (IFileServiceCodePageConverter)_codePageConverters.elementAt(i); 
+			if (codePageConverter.isServerEncodingSupported(serverEncoding, this))
+				return codePageConverter;
+		}
+		}
+		return null;
+	}
 }

rse/plugins/org.eclipse.rse.services/src/org/eclipse/rse/services/files/DefaultFileServiceCodePageConverter.java

@@ -0,0 +1,66 @@
+/********************************************************************************
+ * Copyright (c) 2007 IBM Corporation. All rights reserved.
+ * This program and the accompanying materials are made available under the terms
+ * of the Eclipse Public License v1.0 which accompanies this distribution, and is 
+ * available at http://www.eclipse.org/legal/epl-v10.html
+ * 
+ * Initial Contributors:
+ * The following IBM employees contributed to the Remote System Explorer
+ * component that contains this file: David McKnight.
+ * 
+ * Contributors:
+ * David McKnight    (IBM)  -[209704] [api][dstore] Ability to override default encoding conversion needed.
+ ********************************************************************************/
+package org.eclipse.rse.services.files;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+
+public class DefaultFileServiceCodePageConverter implements
+		IFileServiceCodePageConverter {
+
+	public byte[] convertClientStringToRemoteBytes(String clientString,
+			String remoteEncoding, IFileService fs) {
+		try
+		{
+			return clientString.getBytes(remoteEncoding);
+		}
+		catch (Exception e)
+		{			
+		}
+		return clientString.getBytes();
+	}
+
+	public void convertFileFromRemoteEncoding(File file, String remoteEncoding,
+			String localEncoding, IFileService fs) {
+		
+		// read in the file
+		try
+		{
+			int fileLength = (int)file.length();
+			FileInputStream inputStream = new FileInputStream(file);
+			BufferedInputStream bufInputStream = new BufferedInputStream(inputStream, fileLength);
+			byte[] buffer = new byte[fileLength];
+			int bytesRead = bufInputStream.read(buffer, 0, fileLength);
+			bufInputStream.close();
+			inputStream.close();
+			
+			byte[] localBuffer = new String(buffer, 0, bytesRead, remoteEncoding).getBytes(localEncoding);
+			
+			FileOutputStream outStream = new FileOutputStream(file);		
+			outStream.write(localBuffer, 0, localBuffer.length);			
+		}
+		catch (Exception e)
+		{
+			
+		}
+	}
+
+	public boolean isServerEncodingSupported(String remoteEncoding,
+			IFileService fs) {
+		return true;
+	}
+
+}

rse/plugins/org.eclipse.rse.services/src/org/eclipse/rse/services/files/IFileServiceCodePageConverter.java

@@ -0,0 +1,56 @@
+/********************************************************************************
+ * Copyright (c) 2007 IBM Corporation. All rights reserved.
+ * This program and the accompanying materials are made available under the terms
+ * of the Eclipse Public License v1.0 which accompanies this distribution, and is 
+ * available at http://www.eclipse.org/legal/epl-v10.html
+ * 
+ * Initial Contributors:
+ * The following IBM employees contributed to the Remote System Explorer
+ * component that contains this file: David McKnight.
+ * 
+ * Contributors:
+ * David McKnight    (IBM)  -[209704] [api] Ability to override default encoding conversion needed.
+ ********************************************************************************/
+
+package org.eclipse.rse.services.files;
+
+import java.io.File;
+
+
+/**
+ * This interface is used by the extension point 
+ * It allows overriding the Universal File Subsystem translation of files, and results in 
+ * binary transfer, with calls to the implementor to handle code page conversion.
+ * @since 7.1
+ */
+public interface IFileServiceCodePageConverter {
+
+	/**
+	 * Converts a client string to remote bytes, for use when uploading in binary mode.  
+	 * @param clientString		the client string to convert
+	 * @param remoteEncoding	The remote encoding for the desired server bytes
+	 * @param fs                The file service to apply conversion to.
+	 *                          Can be used to determine implementation specific settings to the converter
+	 * @return					The bytes to upload to the server
+	 */
+	public byte [] convertClientStringToRemoteBytes(String clientString, String remoteEncoding, IFileService fs);
+	
+	/**
+	 * Converts the specified file (which was downloaded from the server in binary mode) from server encoding bytes, to local encoding
+	 * @param file				The file to convert 
+	 * @param localEncoding		The remote encoding of the file
+	 * @param fs                The file service to apply conversion to.
+	 *                          Can be used to determine implementation specific settings to the converter
+	 */
+	public void convertFileFromRemoteEncoding(File file, String remoteEncoding, String localEncoding, IFileService uss);
+	
+	/**
+	 * Indicates whether or not the specified server encoding and subsystem implementation is supported by this code page converter
+	 * @param remoteEncoding		The remote encoding from the server to check
+	 * @param fs                The file service to apply conversion to.
+	 *                          Can be used to determine implementation specific settings to the converter
+	 * @return						True if this code page converter can convert the specified encoding, false otherwise
+	 */
+	public boolean isServerEncodingSupported(String remoteEncoding, IFileService uss);
+	
+}