Concurrent ISVNEditor.openFile Calls Throwing E160024 Conflict SVNException

[cgswtsu78]

Hello,

I'm using SVNKit version 1.8.5. I'm getting several SVN Conflict Exceptions
when my application is under heavy load. I have multiple threads
concurrently trying to commit changes to the same files using the
addorUpdateSVNFiles API below. Could it be that one threads gets an out of
date version of a file when calling editor.openFile since another thread has
just updated that file?

*Code:*
//inputDataMap contains the full file path (key) and Stream representing the
actual file (value).
public long addorUpdateSVNFiles(Map<String, ByteArrayOutputStream>
inputDataMap) throws Exception {
Set<String> filesToAdd = new TreeSet<String>();
Set<String> filesToUpdate = new TreeSet<String>();
SVNRepository repo = null;
ISVNEditor editor = null;
try {
repo = openSession(); //gets a SVN connection from a pool
Iterator<Map.Entry&lt;String, ByteArrayOutputStream>> filesIterator
= inputDataMap.entrySet().iterator();

//loop over all the files being committed and determine if the file
exists or not
while (filesIterator.hasNext()) {
Map.Entry<String, ByteArrayOutputStream> entry =
filesIterator.next();
//key contains the full path to the file and the file name
(/150238966/projecta/150239541.xml)
String key = entry.getKey();
SVNNodeKind nodeKind = repo.checkPath(key, -1);

//if the file exists update else add
if (nodeKind == SVNNodeKind.FILE) {
filesToUpdate.add(key);
} else if (nodeKind == SVNNodeKind.NONE) {
filesToAdd.add(key);
} else {
throw new Exception("Unsupported SVN Node" + nodeKind);
}
}

//filter out directories that exists or that are not directories (files)
LinkedHashSet<String> dirsToCreate = findDirectories(filesToAdd);
for (Iterator<String> dirIterator = dirsToCreate.iterator();
dirIterator.hasNext();) {
String directory = dirIterator.next();
if (directory.contains(XML_FILE_EXTENSION)) {
dirIterator.remove();
} else if (isDirectory(directory, repo)) {
dirIterator.remove();
}
}
editor = repo.getCommitEditor("Adding/Updating Files and
Directories", null);
editor.openRoot(-1);
// create missing directories to support the files being added
for (String folder : dirsToCreate) {
editor.addDir(folder, null, -1);
editor.closeDir();
}
// add new files
for (String file : filesToAdd) {
editor.addFile(file, null, -1);
handleFile(editor, file, inputDataMap);
}
// update existing files
for (String file : filesToUpdate) {
editor.openFile(file, -1);
handleFile(editor, file, inputDataMap);
}
editor.closeEdit();
return 0;
} catch (Exception ex) {
//log error
} finally {
closeSession(repo);
}
}

/**
* findDirectories processes directory paths and returns the unique
directories for SVN to create
* in parent child order.
*/
private LinkedHashSet<String> findDirectories(Set<String> directoryList) {
LinkedHashSet<String> fullSet = new LinkedHashSet<String>();
for (String i : directoryList) {
String[] str = removeLeadingSlash(i).split("\\" + File.separator);
StringBuilder sb = new StringBuilder();
for (String s : str) {
sb.append(s);
fullSet.add(sb.toString());
sb.append(File.separator);
}
}
return fullSet;
}

private void handleFile(ISVNEditor editor, String file, Map<String,
ByteArrayOutputStream> data) throws SVNException {
editor.applyTextDelta(file, null);
SVNDeltaGenerator gen = new SVNDeltaGenerator();
String checksum = gen.sendDelta(file, new
ByteArrayInputStream(data.get(file).toByteArray()), editor, true);
editor.closeFile(file, checksum);
}
}


*StackTrace:*
Timestamp="2015-01-08T06:16:35.184" Logger="Caller+0 at
com.test.SvnTest.addorUpdateSVNFiles(SvnTest.java:200)
" LogLevel="ERROR" ThreadId="contentContainer-5"
Class="SvnTest.java:addorUpdateSVNFiles:200" - SVN Add/Update XML File
Failure, retry attempt failed for username:
org.tmatesoft.svn.core.SVNException: svn: E160024: Conflict at
'/150238966/projecta/150239541.xml'
at
org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:64)
at
org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.java:51)
at
org.tmatesoft.svn.core.internal.io.svn.SVNReader.handleFailureStatus(SVNReader.java:269)
at
org.tmatesoft.svn.core.internal.io.svn.SVNReader.parse(SVNReader.java:248)
at
org.tmatesoft.svn.core.internal.io.svn.SVNConnection.read(SVNConnection.java:272)
at
org.tmatesoft.svn.core.internal.io.svn.SVNCommitEditor.closeEdit(SVNCommitEditor.java:203)
at com.test.SvnTest.addorUpdateSVNFiles(SvnTest.java:192)
at com.test.create(CreateImpl.java:447)
at sun.reflect.GeneratedMethodAccessor273.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:190)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157)
at
org.springframework.transaction.interceptor.TransactionInterceptor$1.proceedWithInvocation(TransactionInterceptor.java:98)
at
org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:262)
at
org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:95)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at
org.springframework.transaction.interceptor.TransactionInterceptor$1.proceedWithInvocation(TransactionInterceptor.java:98)
at
org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:262)
at
org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:95)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at
org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:92)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at
org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:207)
at com.sun.proxy.$Proxy94.create(Unknown Source)
at com.test.TestListener.onMessage(TestListener.java:38)
at
org.springframework.jms.listener.adapter.MessageListenerAdapter.onMessage(MessageListenerAdapter.java:341)
at
org.springframework.jms.listener.AbstractMessageListenerContainer.doInvokeListener(AbstractMessageListenerContainer.java:537)
at
org.springframework.jms.listener.AbstractMessageListenerContainer.invokeListener(AbstractMessageListenerContainer.java:497)
at
org.springframework.jms.listener.AbstractMessageListenerContainer.doExecuteListener(AbstractMessageListenerContainer.java:468)
at
org.springframework.jms.listener.AbstractPollingMessageListenerContainer.doReceiveAndExecute(AbstractPollingMessageListenerContainer.java:325)
at
org.springframework.jms.listener.AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:263)
at
org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1102)
at
org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.executeOngoingLoop(DefaultMessageListenerContainer.java:1094)
at
org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:991)
at java.lang.Thread.run(Thread.java:662)



--
View this message in context: http://subversion.1072662.n5.nabble.com/Concurrent-ISVNEditor-openFile-Calls-Throwing-E160024-Conflict-SVNException-tp191424.html
Sent from the SVNKit - Users mailing list archive at Nabble.com.

[Dmitry Pavlenko]

Hello dear user,
Sorry for the late reply.

First of all, if you use SVNKit in multithreaded environment, please have a look at these posts

http://vcs.atspace.co.uk/2012/09/21/are-svnkit-methods-reenterable/
http://vcs.atspace.co.uk/2012/09/22/what-svnkit-resources-should-be-disposed/

Second, the code you provided is too big to analyze, could you please send us minimal code that
creates a SVN repository and reproduces the problem on it OR please log all
openFile/closeFile/addFile/openDir/... and so on calls with argument they have. This could help us
to understand if you have correct calls sequence. From what I see, you have no closeDir() call
corresponding to openRoot(), but it should be there. You can read this post about correct calls
sequence:

http://vcs.atspace.co.uk/2012/07/20/subversion-remote-api-committing-without-working-copy/

--
Dmitry Pavlenko,
TMate Software,
http://subgit.com/ - git-svn bridge

Post by cgswtsu78
Hello,
I'm using SVNKit version 1.8.5. I'm getting several SVN Conflict
Exceptions when my application is under heavy load. I have multiple
threads
concurrently trying to commit changes to the same files using the
addorUpdateSVNFiles API below. Could it be that one threads gets an out of
date version of a file when calling editor.openFile since another thread
has just updated that file?
*Code:*
//inputDataMap contains the full file path (key) and Stream representing
the actual file (value).
public long addorUpdateSVNFiles(Map<String, ByteArrayOutputStream>
inputDataMap) throws Exception {
Set<String> filesToAdd = new TreeSet<String>();
Set<String> filesToUpdate = new TreeSet<String>();
SVNRepository repo = null;
ISVNEditor editor = null;
try {
repo = openSession(); //gets a SVN connection from a pool
Iterator<Map.Entry&lt;String, ByteArrayOutputStream>> filesIterator
= inputDataMap.entrySet().iterator();
//loop over all the files being committed and determine if the file
exists or not
while (filesIterator.hasNext()) {
Map.Entry<String, ByteArrayOutputStream> entry =
filesIterator.next();
//key contains the full path to the file and the file name
(/150238966/projecta/150239541.xml)
String key = entry.getKey();
SVNNodeKind nodeKind = repo.checkPath(key, -1);
//if the file exists update else add
if (nodeKind == SVNNodeKind.FILE) {
filesToUpdate.add(key);
} else if (nodeKind == SVNNodeKind.NONE) {
filesToAdd.add(key);
} else {
throw new Exception("Unsupported SVN Node" + nodeKind);
}
}
//filter out directories that exists or that are not directories (files)
LinkedHashSet<String> dirsToCreate = findDirectories(filesToAdd);
for (Iterator<String> dirIterator = dirsToCreate.iterator();
dirIterator.hasNext();) {
String directory = dirIterator.next();
if (directory.contains(XML_FILE_EXTENSION)) {
dirIterator.remove();
} else if (isDirectory(directory, repo)) {
dirIterator.remove();
}
}
editor = repo.getCommitEditor("Adding/Updating Files and
Directories", null);
editor.openRoot(-1);
// create missing directories to support the files being added
for (String folder : dirsToCreate) {
editor.addDir(folder, null, -1);
editor.closeDir();
}
// add new files
for (String file : filesToAdd) {
editor.addFile(file, null, -1);
handleFile(editor, file, inputDataMap);
}
// update existing files
for (String file : filesToUpdate) {
editor.openFile(file, -1);
handleFile(editor, file, inputDataMap);
}
editor.closeEdit();
return 0;
} catch (Exception ex) {
//log error
} finally {
closeSession(repo);
}
}
/**
* findDirectories processes directory paths and returns the unique
directories for SVN to create
* in parent child order.
*/
private LinkedHashSet<String> findDirectories(Set<String> directoryList)
{ LinkedHashSet<String> fullSet = new LinkedHashSet<String>();
for (String i : directoryList) {
String[] str = removeLeadingSlash(i).split("\\" + File.separator);
StringBuilder sb = new StringBuilder();
for (String s : str) {
sb.append(s);
fullSet.add(sb.toString());
sb.append(File.separator);
}
}
return fullSet;
}
private void handleFile(ISVNEditor editor, String file, Map<String,
ByteArrayOutputStream> data) throws SVNException {
editor.applyTextDelta(file, null);
SVNDeltaGenerator gen = new SVNDeltaGenerator();
String checksum = gen.sendDelta(file, new
ByteArrayInputStream(data.get(file).toByteArray()), editor, true);
editor.closeFile(file, checksum);
}
}
*StackTrace:*
Timestamp="2015-01-08T06:16:35.184" Logger="Caller+0 at
com.test.SvnTest.addorUpdateSVNFiles(SvnTest.java:200)
" LogLevel="ERROR" ThreadId="contentContainer-5"
Class="SvnTest.java:addorUpdateSVNFiles:200" - SVN Add/Update XML File
org.tmatesoft.svn.core.SVNException: svn: E160024: Conflict at
'/150238966/projecta/150239541.xml'
at
org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.ja
va:64) at
org.tmatesoft.svn.core.internal.wc.SVNErrorManager.error(SVNErrorManager.ja
va:51) at
org.tmatesoft.svn.core.internal.io.svn.SVNReader.handleFailureStatus(SVNRea
der.java:269) at
org.tmatesoft.svn.core.internal.io.svn.SVNReader.parse(SVNReader.java:248)
at
org.tmatesoft.svn.core.internal.io.svn.SVNConnection.read(SVNConnection.jav
a:272) at
org.tmatesoft.svn.core.internal.io.svn.SVNCommitEditor.closeEdit(SVNCommitE
ditor.java:203) at com.test.SvnTest.addorUpdateSVNFiles(SvnTest.java:192)
at com.test.create(CreateImpl.java:447)
at sun.reflect.GeneratedMethodAccessor273.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImp
l.java:25) at java.lang.reflect.Method.invoke(Method.java:597)
at
org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(Aop
Utils.java:317) at
org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoin
t(ReflectiveMethodInvocation.java:190) at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(Reflec
tiveMethodInvocation.java:157) at
org.springframework.transaction.interceptor.TransactionInterceptor$1.procee
dWithInvocation(TransactionInterceptor.java:98) at
org.springframework.transaction.interceptor.TransactionAspectSupport.invoke
WithinTransaction(TransactionAspectSupport.java:262) at
org.springframework.transaction.interceptor.TransactionInterceptor.invoke(T
ransactionInterceptor.java:95) at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(Reflec
tiveMethodInvocation.java:179) at
org.springframework.transaction.interceptor.TransactionInterceptor$1.procee
dWithInvocation(TransactionInterceptor.java:98) at
org.springframework.transaction.interceptor.TransactionAspectSupport.invoke
WithinTransaction(TransactionAspectSupport.java:262) at
org.springframework.transaction.interceptor.TransactionInterceptor.invoke(T
ransactionInterceptor.java:95) at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(Reflec
tiveMethodInvocation.java:179) at
org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(Expo
seInvocationInterceptor.java:92) at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(Reflec
tiveMethodInvocation.java:179) at
org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopPr
oxy.java:207) at com.sun.proxy.$Proxy94.create(Unknown Source)
at com.test.TestListener.onMessage(TestListener.java:38)
at
org.springframework.jms.listener.adapter.MessageListenerAdapter.onMessage(M
essageListenerAdapter.java:341) at
org.springframework.jms.listener.AbstractMessageListenerContainer.doInvokeL
istener(AbstractMessageListenerContainer.java:537) at
org.springframework.jms.listener.AbstractMessageListenerContainer.invokeLis
tener(AbstractMessageListenerContainer.java:497) at
org.springframework.jms.listener.AbstractMessageListenerContainer.doExecute
Listener(AbstractMessageListenerContainer.java:468) at
org.springframework.jms.listener.AbstractPollingMessageListenerContainer.do
ReceiveAndExecute(AbstractPollingMessageListenerContainer.java:325) at
org.springframework.jms.listener.AbstractPollingMessageListenerContainer.re
ceiveAndExecute(AbstractPollingMessageListenerContainer.java:263) at
org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessa
geListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1102)
at
org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessa
geListenerInvoker.executeOngoingLoop(DefaultMessageListenerContainer.java:1
094) at
org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessa
geListenerInvoker.run(DefaultMessageListenerContainer.java:991) at
java.lang.Thread.run(Thread.java:662)
--
http://subversion.1072662.n5.nabble.com/Concurrent-ISVNEditor-openFile-Cal
ls-Throwing-E160024-Conflict-SVNException-tp191424.html Sent from the
SVNKit - Users mailing list archive at Nabble.com.